From 0243f671ed30c6d3aaecaf94d755a78d471a07fa Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 21 Jun 2020 20:48:56 +0100 Subject: [PATCH 01/86] Added Directory Service #85 --- .../provider/vpc/resource/identity.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 cloudiscovery/provider/vpc/resource/identity.py diff --git a/cloudiscovery/provider/vpc/resource/identity.py b/cloudiscovery/provider/vpc/resource/identity.py new file mode 100644 index 0000000..7aa70ce --- /dev/null +++ b/cloudiscovery/provider/vpc/resource/identity.py @@ -0,0 +1,66 @@ +from typing import List + +from provider.vpc.command import VpcOptions +from shared.common import ( + ResourceProvider, + Resource, + message_handler, + ResourceDigest, + ResourceEdge, + resource_tags, +) +from shared.error_handler import exception + + +class DIRECTORYSERVICE(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Directory service + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + + client = self.vpc_options.client("ds") + + resources_found = [] + + response = client.describe_directories() + + message_handler("Collecting data from Directory Services...", "HEADER") + + if len(response["DirectoryDescriptions"]) > 0: + + for data in response["DirectoryDescriptions"]: + + if "VpcSettings" in data: + + if data["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: + directory_service_digest = ResourceDigest( + id=data["DirectoryId"], type="aws_ds" + ) + resources_found.append( + Resource( + digest=directory_service_digest, + name=data["Name"], + details="", + group="identity", + tags=resource_tags(data), + ) + ) + + for subnet in data["VpcSettings"]["SubnetIds"]: + self.relations_found.append( + ResourceEdge( + from_node=directory_service_digest, + to_node=ResourceDigest( + id=subnet, type="aws_subnet" + ), + ) + ) + + return resources_found From 287c2f0cbd4dc208964135664d7436a39e275db1 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 21 Jun 2020 21:14:16 +0100 Subject: [PATCH 02/86] Added Workspace #85 --- .../provider/vpc/resource/enduser.py | 82 +++++++++++++++++++ cloudiscovery/shared/common.py | 2 +- 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 cloudiscovery/provider/vpc/resource/enduser.py diff --git a/cloudiscovery/provider/vpc/resource/enduser.py b/cloudiscovery/provider/vpc/resource/enduser.py new file mode 100644 index 0000000..fa012de --- /dev/null +++ b/cloudiscovery/provider/vpc/resource/enduser.py @@ -0,0 +1,82 @@ +from typing import List + +from provider.vpc.command import VpcOptions +from shared.common import ( + ResourceProvider, + Resource, + message_handler, + ResourceDigest, + ResourceEdge, + resource_tags, + get_name_tag, +) +from shared.error_handler import exception + + +class WORKSPACES(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Workspaces + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + + client = self.vpc_options.client("workspaces") + + resources_found = [] + + response = client.describe_workspaces() + + message_handler("Collecting data from Workspaces...", "HEADER") + + if len(response["Workspaces"]) > 0: + + for data in response["Workspaces"]: + + # Get tag name + tags = client.describe_tags(ResourceId=data["WorkspaceId"]) + nametag = get_name_tag(tags) + + workspace_name = data["WorkspaceId"] if nametag is None else nametag + + directory_service = self.vpc_options.client("ds") + directories = directory_service.describe_directories( + DirectoryIds=[data["DirectoryId"]] + ) + + for directorie in directories["DirectoryDescriptions"]: + + if "VpcSettings" in directorie: + + if ( + directorie["VpcSettings"]["VpcId"] + == self.vpc_options.vpc_id + ): + workspace_digest = ResourceDigest( + id=data["WorkspaceId"], type="aws_workspaces" + ) + resources_found.append( + Resource( + digest=workspace_digest, + name=workspace_name, + details="", + group="enduser", + tags=resource_tags(tags), + ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=workspace_digest, + to_node=ResourceDigest( + id=directorie["DirectoryId"], type="aws_ds" + ), + ) + ) + + return resources_found diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index c76e98f..8fca3cf 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -159,7 +159,7 @@ def get_name_tag(d) -> Optional[str]: def get_tag(d, tag_name) -> Optional[str]: for k, v in d.items(): - if k == "Tags": + if k in ("Tags", "TagList"): for value in v: if value["Key"] == tag_name: return value["Value"] From 35ab9aad25275ca256d90c737761dd0072998e82 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 21 Jun 2020 23:13:30 +0100 Subject: [PATCH 03/86] Added QuickSight data source #85 --- .../provider/vpc/resource/analytics.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 85cc8ad..b6e5080 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -147,3 +147,71 @@ def get_resources(self) -> List[Resource]: break return resources_found + + +class QUICKSIGHT(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Quicksight + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + + client = self.vpc_options.client("quicksight") + + resources_found = [] + + # Get accountid + account_id = self.vpc_options.account_number() + + response = client.list_data_sources(AwsAccountId=account_id) + + message_handler("Collecting data from Quicksight...", "HEADER") + + if len(response["DataSources"]) > 0: + + for data in response["DataSources"]: + + # Twitter and S3 data source is not supported + if data["Type"] not in ("TWITTER", "S3"): + + data_source = client.describe_data_source( + AwsAccountId=account_id, DataSourceId=data["DataSourceId"] + ) + + if "VpcConnectionProperties" in data_source: + + if ( + self.vpc_options.vpc_id + in data_source["VpcConnectionProperties"][ + "VpcConnectionArn" + ] + ): + quicksight_digest = ResourceDigest( + id=data["DataSourceId"], type="aws_quicksight" + ) + resources_found.append( + Resource( + digest=quicksight_digest, + name=data["DataSourceId"], + details="", + group="analytics", + tags=resource_tags(data), + ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=quicksight_digest, + to_node=ResourceDigest( + id=self.vpc_options.vpc_id, type="aws_vpc" + ), + ) + ) + + return resources_found From 3fb5b9fdf75d8cf42680215d59154f04f47e2a01 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 22 Jun 2020 22:03:05 +0700 Subject: [PATCH 04/86] added codecov --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a878033..201884e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: python: circleci/python@0.3.0 + codecov: codecov/codecov@1.1.0 jobs: build: @@ -20,6 +21,7 @@ jobs: path: test-results - store_artifacts: path: test-results + - codecov/upload deploy: executor: python/default steps: From 7a0ea935c1f6d95f5456f6f030983175888e9bad Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 22 Jun 2020 22:09:20 +0700 Subject: [PATCH 05/86] added codecov badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9ab0414..e7849ab 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![PyPI version](https://badge.fury.io/py/cloudiscovery.svg)](https://badge.fury.io/py/cloudiscovery) [![Downloads](https://pepy.tech/badge/cloudiscovery)](https://pepy.tech/project/cloudiscovery) +[![codecov](https://codecov.io/gh/Cloud-Architects/cloudiscovery/branch/develop/graph/badge.svg)](https://codecov.io/gh/Cloud-Architects/cloudiscovery) ![python version](https://img.shields.io/badge/python-3.6%2C3.7%2C3.8-blue?logo=python) [![CircleCI](https://circleci.com/gh/Cloud-Architects/cloudiscovery.svg?style=svg)](https://circleci.com/gh/Cloud-Architects/cloudiscovery) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/c0a7a5bc51044c7ca8bd9115965e4467)](https://www.codacy.com/gh/Cloud-Architects/cloudiscovery?utm_source=github.com&utm_medium=referral&utm_content=Cloud-Architects/cloudiscovery&utm_campaign=Badge_Grade) From 3b47c90f62ef21895309dc0bcd9e0a29eaa8358b Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 22 Jun 2020 22:38:19 +0700 Subject: [PATCH 06/86] excluded tests from code coverage --- .coveragerc | 2 +- .../provider/vpc/resource/network.py | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 1f33683..2c331ad 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -omit = tests/*,venv/* \ No newline at end of file +omit = cloudiscovery/tests/*,venv/* \ No newline at end of file diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index 2952e0b..3f2a3db 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -678,3 +678,26 @@ def analyze_restapi(self, data): ), ) return False, None + + +class VpnConnection(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Vpn Connections + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + client = self.vpc_options.client("vpc") + vpc_response = client.describe_vpcs(VpcIds=[self.vpc_options.vpc_id]) + return [ + Resource( + digest=self.vpc_options.vpc_digest(), + name=self.vpc_options.vpc_id, + tags=resource_tags(vpc_response["Vpcs"][0]), + ) + ] From 3bd1af836176424863299331f338f959422b5589 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 22 Jun 2020 22:46:47 +0700 Subject: [PATCH 07/86] excluded tests from code coverage --- .../provider/vpc/resource/network.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index 3f2a3db..2952e0b 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -678,26 +678,3 @@ def analyze_restapi(self, data): ), ) return False, None - - -class VpnConnection(ResourceProvider): - def __init__(self, vpc_options: VpcOptions): - """ - Vpn Connections - - :param vpc_options: - """ - super().__init__() - self.vpc_options = vpc_options - - @exception - def get_resources(self) -> List[Resource]: - client = self.vpc_options.client("vpc") - vpc_response = client.describe_vpcs(VpcIds=[self.vpc_options.vpc_id]) - return [ - Resource( - digest=self.vpc_options.vpc_digest(), - name=self.vpc_options.vpc_id, - tags=resource_tags(vpc_response["Vpcs"][0]), - ) - ] From 27472f31f26a0383d33ced6a42f86ad113a573cf Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 00:12:49 +0700 Subject: [PATCH 08/86] #7 added VPN resources --- .../provider/vpc/resource/network.py | 160 +++++++++++++++++- cloudiscovery/shared/diagram.py | 4 + 2 files changed, 162 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index 2952e0b..aa72f80 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -327,7 +327,6 @@ def __init__(self, vpc_options: VpcOptions): @exception def get_resources(self) -> List[Resource]: - client = self.vpc_options.client("ec2") resources_found = [] @@ -432,7 +431,6 @@ def __init__(self, vpc_options: VpcOptions): @exception def get_resources(self) -> List[Resource]: - client = self.vpc_options.client("ec2") resources_found = [] @@ -678,3 +676,161 @@ def analyze_restapi(self, data): ), ) return False, None + + +class VpnConnection(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Vpn Connections + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + client = self.vpc_options.client("ec2") + vpn_response = client.describe_vpn_connections() + resources: List[Resource] = [] + + for vpn_connection in vpn_response["VpnConnections"]: + if ( + "VpnGatewayId" in vpn_connection + and vpn_connection["VpnGatewayId"] != "" + ): + vpn_gateway_id = vpn_connection["VpnGatewayId"] + vpn_gateway_response = client.describe_vpn_gateways( + Filters=[ + { + "Name": "attachment.vpc-id", + "Values": [self.vpc_options.vpc_id,], + } + ], + VpnGatewayIds=[vpn_gateway_id], + ) + if len(vpn_gateway_response["VpnGateways"]) > 0: + connection_digest = ResourceDigest( + id=vpn_connection["VpnConnectionId"], type="aws_vpn_connection" + ) + vpn_nametag = get_name_tag(vpn_connection) + vpn_name = ( + vpn_connection["VpnConnectionId"] + if vpn_nametag is None + else vpn_nametag + ) + resources.append( + Resource( + digest=connection_digest, + name=vpn_name, + group="network", + tags=resource_tags(vpn_connection), + ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=connection_digest, + to_node=self.vpc_options.vpc_digest(), + ) + ) + + vpn_gateway_digest = ResourceDigest( + id=vpn_gateway_id, type="aws_vpn_gateway" + ) + vgw_nametag = get_name_tag(vpn_gateway_response["VpnGateways"][0]) + vgw_name = vpn_gateway_id if vgw_nametag is None else vgw_nametag + resources.append( + Resource( + digest=vpn_gateway_digest, + name=vgw_name, + group="network", + tags=resource_tags(vpn_gateway_response["VpnGateways"][0]), + ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=connection_digest, to_node=vpn_gateway_digest + ) + ) + + if ( + "CustomerGatewayId" in vpn_connection + and vpn_connection["CustomerGatewayId"] != "" + ): + self.add_customer_gateway( + client, connection_digest, resources, vpn_connection + ) + + return resources + + def add_customer_gateway( + self, client, connection_digest, resources, vpn_connection + ): + customer_gateway_id = vpn_connection["CustomerGatewayId"] + vcw_gateway_response = client.describe_customer_gateways( + CustomerGatewayIds=[customer_gateway_id] + ) + vcw_gateway_digest = ResourceDigest( + id=customer_gateway_id, type="aws_customer_gateway" + ) + vcw_nametag = get_name_tag(vcw_gateway_response["CustomerGateways"][0]) + vcw_name = customer_gateway_id if vcw_nametag is None else vcw_nametag + resources.append( + Resource( + digest=vcw_gateway_digest, + name=vcw_name, + group="network", + tags=resource_tags(vcw_gateway_response["CustomerGateways"][0]), + ) + ) + self.relations_found.append( + ResourceEdge(from_node=connection_digest, to_node=vcw_gateway_digest) + ) + + +class VpnClientEndpoint(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Vpn Client Endpoints + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + client = self.vpc_options.client("ec2") + client_vpn_endpoints = client.describe_client_vpn_endpoints() + resources: List[Resource] = [] + + for client_vpn_endpoint in client_vpn_endpoints["ClientVpnEndpoints"]: + if client_vpn_endpoint["VpcId"] == self.vpc_options.vpc_id: + digest = ResourceDigest( + id=client_vpn_endpoint["ClientVpnEndpointId"], + type="aws_vpn_client_endpoint", + ) + nametag = get_name_tag(client_vpn_endpoint) + name = ( + client_vpn_endpoint["ClientVpnEndpointId"] + if nametag is None + else nametag + ) + resources.append( + Resource( + digest=digest, + name=name, + group="network", + tags=resource_tags(client_vpn_endpoint), + ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=digest, to_node=self.vpc_options.vpc_digest() + ) + ) + + return resources diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 34671b2..974093a 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -214,6 +214,10 @@ class Mapsources: "aws_iotsitewise": "IotSitewise", "aws_neptune_cluster": "Neptune", "aws_alexa_for_business": "AlexaForBusiness", + "aws_customer_gateway": "SiteToSiteVpn", + "aws_vpn_connection": "SiteToSiteVpn", + "aws_vpn_gateway": "SiteToSiteVpn", + "aws_vpn_client_endpoint": "ClientVpn", } From 0e73c63efa202d16eb433ad8971122a1bc44d171 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 00:22:44 +0700 Subject: [PATCH 09/86] improved readme --- README.md | 66 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e7849ab..4a86695 100644 --- a/README.md +++ b/README.md @@ -22,40 +22,44 @@ Example of a diagram: Following resources are checked in VPC command: +* Autoscaling Group +* Classic/Network/Application Load Balancer +* Client VPN Endpoints +* CloudHSM +* DocumentDB * EC2 Instance -* IAM Policy -* Lambda -* RDS -* EFS +* ECS +* EFS * ElastiCache -* S3 Policy * Elasticsearch -* DocumentDB -* SQS Queue Policy -* MSK -* NAT Gateway -* Internet Gateway (IGW) -* Classic/Network/Application Load Balancer -* Route Table -* Subnet -* NACL -* Security Group -* VPC Peering -* VPC Endpoint * EKS -* Synthetic Canary * EMR -* ECS -* Autoscaling Group +* IAM Policy +* Internet Gateway (IGW) +* Lambda * Media Connect * Media Live * Media Store Policy -* REST Api Policy +* MSK +* NACL +* NAT Gateway * Neptune -* CloudHSM +* RDS +* REST Api Policy +* Route Table +* S3 Policy * Sagemaker Notebook * Sagemaker Training Job * Sagemaker Model +* Security Group +* SQS Queue Policy +* Site-to-Site VPN Connections +* Subnet +* Synthetic Canary +* VPC Peering +* VPC Endpoint +* VPN Customer Gateways +* Virtual Private Gateways The subnets are aggregated to simplify the diagram and hide infrastructure redundancies. There can be two types of subnet aggregates: 1. Private* ones with a route `0.0.0.0/0` to Internet Gateway @@ -71,15 +75,15 @@ Example of a diagram: Following resources are checked in Policy command: -* IAM User +* [AWS Principal](https://gist.github.com/shortjared/4c1e3fe52bdfa47522cfe5b41e5d6f22) that are able to assume roles * IAM Group +* IAM Group to policy relationship * IAM Policy -* IAM Roles +* IAM Role +* IAM Role to policy relationship +* IAM User * IAM User to group relationship * IAM User to policy relationship -* IAM Group to policy relationship -* IAM Role to policy relationship -* [AWS Principals](https://gist.github.com/shortjared/4c1e3fe52bdfa47522cfe5b41e5d6f22) that are able to assume roles Some roles can be aggregated to simplify the diagram. If a role is associated with a principal and is not attached to any named policy, will be aggregated. @@ -91,12 +95,12 @@ Example of a diagram: Following resources are checked in IoT command: -* IoT Thing -* IoT Thing Type * IoT Billing Group -* IoT Policies -* IoT Jobs * IoT Certificates +* IoT Jobs +* IoT Policies +* IoT Thing +* IoT Thing Type ## Requirements and Installation From bb286c99a037c8fedd45fd2233cb16a7e0df2d75 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 00:28:30 +0700 Subject: [PATCH 10/86] pylint improvements --- .pylintrc | 2 +- cloudiscovery/provider/vpc/resource/network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 7cb9b0c..bab35f2 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,4 +5,4 @@ init-hook='import sys; sys.path.append("cloudiscovery")' max-line-length=120 [MESSAGES CONTROL] -disable=pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches +disable=missing-docstring,pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index aa72f80..ca82e99 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -709,7 +709,7 @@ def get_resources(self) -> List[Resource]: ], VpnGatewayIds=[vpn_gateway_id], ) - if len(vpn_gateway_response["VpnGateways"]) > 0: + if vpn_gateway_response["VpnGateways"]: connection_digest = ResourceDigest( id=vpn_connection["VpnConnectionId"], type="aws_vpn_connection" ) From 07824829f226e5da9dc92d13274b1b798f73baf5 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 18:58:13 +0100 Subject: [PATCH 11/86] Added quicksight RDS connection #85 --- .../provider/vpc/resource/analytics.py | 219 ++++++++++-------- .../provider/vpc/resource/database.py | 52 +++-- 2 files changed, 147 insertions(+), 124 deletions(-) diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index b6e5080..9ce8e60 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -2,6 +2,7 @@ from typing import List from provider.vpc.command import VpcOptions, check_ipvpc_inpolicy +from provider.vpc.resource.database import RDS from shared.common import ( datetime_to_string, ResourceProvider, @@ -35,54 +36,52 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Elasticsearch Domains...", "HEADER") - if len(response["DomainNames"]) > 0: + for data in response["DomainNames"]: - for data in response["DomainNames"]: + elasticsearch_domain = client.describe_elasticsearch_domain( + DomainName=data["DomainName"] + ) - elasticsearch_domain = client.describe_elasticsearch_domain( - DomainName=data["DomainName"] - ) + documentpolicy = elasticsearch_domain["DomainStatus"]["AccessPolicies"] - documentpolicy = elasticsearch_domain["DomainStatus"]["AccessPolicies"] + document = json.dumps(documentpolicy, default=datetime_to_string) - document = json.dumps(documentpolicy, default=datetime_to_string) + # check either vpc_id or potencial subnet ip are found + ipvpc_found = check_ipvpc_inpolicy( + document=document, vpc_options=self.vpc_options + ) - # check either vpc_id or potencial subnet ip are found - ipvpc_found = check_ipvpc_inpolicy( - document=document, vpc_options=self.vpc_options + # elasticsearch uses accesspolicies too, so check both situation + if ( + elasticsearch_domain["DomainStatus"]["VPCOptions"]["VPCId"] + == self.vpc_options.vpc_id + or ipvpc_found is True + ): + list_tags_response = client.list_tags( + ARN=elasticsearch_domain["DomainStatus"]["ARN"] ) - - # elasticsearch uses accesspolicies too, so check both situation - if ( - elasticsearch_domain["DomainStatus"]["VPCOptions"]["VPCId"] - == self.vpc_options.vpc_id - or ipvpc_found is True - ): - list_tags_response = client.list_tags( - ARN=elasticsearch_domain["DomainStatus"]["ARN"] - ) - digest = ResourceDigest( - id=elasticsearch_domain["DomainStatus"]["DomainId"], - type="aws_elasticsearch_domain", + digest = ResourceDigest( + id=elasticsearch_domain["DomainStatus"]["DomainId"], + type="aws_elasticsearch_domain", + ) + resources_found.append( + Resource( + digest=digest, + name=elasticsearch_domain["DomainStatus"]["DomainName"], + details="", + group="analytics", + tags=resource_tags(list_tags_response), ) - resources_found.append( - Resource( - digest=digest, - name=elasticsearch_domain["DomainStatus"]["DomainName"], - details="", - group="analytics", - tags=resource_tags(list_tags_response), + ) + for subnet_id in elasticsearch_domain["DomainStatus"]["VPCOptions"][ + "SubnetIds" + ]: + self.relations_found.append( + ResourceEdge( + from_node=digest, + to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), ) ) - for subnet_id in elasticsearch_domain["DomainStatus"]["VPCOptions"][ - "SubnetIds" - ]: - self.relations_found.append( - ResourceEdge( - from_node=digest, - to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), - ) - ) return resources_found @@ -108,44 +107,42 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from MSK Clusters...", "HEADER") - if len(response["ClusterInfoList"]) > 0: - - # iterate cache clusters to get subnet groups - for data in response["ClusterInfoList"]: + # iterate cache clusters to get subnet groups + for data in response["ClusterInfoList"]: - msk_subnets = ", ".join(data["BrokerNodeGroupInfo"]["ClientSubnets"]) + msk_subnets = ", ".join(data["BrokerNodeGroupInfo"]["ClientSubnets"]) - ec2 = self.vpc_options.session.resource( - "ec2", region_name=self.vpc_options.region_name - ) + ec2 = self.vpc_options.session.resource( + "ec2", region_name=self.vpc_options.region_name + ) - filters = [{"Name": "vpc-id", "Values": [self.vpc_options.vpc_id]}] + filters = [{"Name": "vpc-id", "Values": [self.vpc_options.vpc_id]}] - subnets = ec2.subnets.filter(Filters=filters) + subnets = ec2.subnets.filter(Filters=filters) - for subnet in list(subnets): + for subnet in list(subnets): - if subnet.id in msk_subnets: - digest = ResourceDigest( - id=data["ClusterArn"], type="aws_msk_cluster" - ) - resources_found.append( - Resource( - digest=digest, - name=data["ClusterName"], - details="", - group="analytics", - tags=resource_tags(data), - ) + if subnet.id in msk_subnets: + digest = ResourceDigest( + id=data["ClusterArn"], type="aws_msk_cluster" + ) + resources_found.append( + Resource( + digest=digest, + name=data["ClusterName"], + details="", + group="analytics", + tags=resource_tags(data), ) - self.relations_found.append( - ResourceEdge( - from_node=digest, - to_node=ResourceDigest(id=subnet.id, type="aws_subnet"), - ) + ) + self.relations_found.append( + ResourceEdge( + from_node=digest, + to_node=ResourceDigest(id=subnet.id, type="aws_subnet"), ) + ) - break + break return resources_found @@ -173,45 +170,69 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Quicksight...", "HEADER") - if len(response["DataSources"]) > 0: + for data in response["DataSources"]: - for data in response["DataSources"]: + # Twitter and S3 data source is not supported + if data["Type"] not in ("TWITTER", "S3"): - # Twitter and S3 data source is not supported - if data["Type"] not in ("TWITTER", "S3"): + data_source = client.describe_data_source( + AwsAccountId=account_id, DataSourceId=data["DataSourceId"] + ) - data_source = client.describe_data_source( - AwsAccountId=account_id, DataSourceId=data["DataSourceId"] - ) + if "RdsParameters" in data_source["DataSource"]["DataSourceParameters"]: - if "VpcConnectionProperties" in data_source: + instance_id = data_source["DataSource"]["DataSourceParameters"][ + "RdsParameters" + ]["InstanceId"] + rds = RDS(self.vpc_options).get_resources(instance_id=instance_id) - if ( - self.vpc_options.vpc_id - in data_source["VpcConnectionProperties"][ - "VpcConnectionArn" - ] - ): - quicksight_digest = ResourceDigest( - id=data["DataSourceId"], type="aws_quicksight" + if len(rds) > 0: + + quicksight_digest = ResourceDigest( + id=data["DataSourceId"], type="aws_quicksight" + ) + resources_found.append( + Resource( + digest=quicksight_digest, + name=data["DataSourceId"], + details="", + group="analytics", + tags=resource_tags(data), ) - resources_found.append( - Resource( - digest=quicksight_digest, - name=data["DataSourceId"], - details="", - group="analytics", - tags=resource_tags(data), - ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=quicksight_digest, to_node=rds[0].digest, ) + ) - self.relations_found.append( - ResourceEdge( - from_node=quicksight_digest, - to_node=ResourceDigest( - id=self.vpc_options.vpc_id, type="aws_vpc" - ), - ) + if "VpcConnectionProperties" in data_source: + + if ( + self.vpc_options.vpc_id + in data_source["VpcConnectionProperties"]["VpcConnectionArn"] + ): + quicksight_digest = ResourceDigest( + id=data["DataSourceId"], type="aws_quicksight" + ) + resources_found.append( + Resource( + digest=quicksight_digest, + name=data["DataSourceId"], + details="", + group="analytics", + tags=resource_tags(data), ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=quicksight_digest, + to_node=ResourceDigest( + id=self.vpc_options.vpc_id, type="aws_vpc" + ), + ) + ) return resources_found diff --git a/cloudiscovery/provider/vpc/resource/database.py b/cloudiscovery/provider/vpc/resource/database.py index ab4e8d3..373d548 100644 --- a/cloudiscovery/provider/vpc/resource/database.py +++ b/cloudiscovery/provider/vpc/resource/database.py @@ -23,37 +23,39 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception - def get_resources(self) -> List[Resource]: + def get_resources(self, instance_id=None) -> List[Resource]: client = self.vpc_options.client("rds") + params = { + "Name": "engine", + "Values": [ + "aurora", + "aurora-mysql", + "aurora-postgresql", + "mariadb", + "mysql", + "oracle-ee", + "oracle-se2", + "oracle-se1", + "oracle-se", + "postgres", + "sqlserver-ee", + "sqlserver-se", + "sqlserver-ex", + "sqlserver-web", + ], + } + + if instance_id is not None: + params.update({"Name": "db-instance-id", "Values": [instance_id]}) + resources_found = [] - response = client.describe_db_instances( - Filters=[ - { - "Name": "engine", - "Values": [ - "aurora", - "aurora-mysql", - "aurora-postgresql", - "mariadb", - "mysql", - "oracle-ee", - "oracle-se2", - "oracle-se1", - "oracle-se", - "postgres", - "sqlserver-ee", - "sqlserver-se", - "sqlserver-ex", - "sqlserver-web", - ], - } - ] - ) + response = client.describe_db_instances(Filters=[params]) - message_handler("Collecting data from RDS Instances...", "HEADER") + if instance_id is None: + message_handler("Collecting data from RDS Instances...", "HEADER") for data in response["DBInstances"]: if data["DBSubnetGroup"]["VpcId"] == self.vpc_options.vpc_id: From 09534ce7b56c45153c4609238b6c139e592260ce Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 19:07:38 +0100 Subject: [PATCH 12/86] Added quicksight RDS connection #85 --- README.md | 3 +++ cloudiscovery/provider/vpc/resource/analytics.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a86695..aeaa27b 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Following resources are checked in VPC command: * Client VPN Endpoints * CloudHSM * DocumentDB +* Directory Service * EC2 Instance * ECS * EFS @@ -44,6 +45,7 @@ Following resources are checked in VPC command: * NACL * NAT Gateway * Neptune +* QuickSight * RDS * REST Api Policy * Route Table @@ -60,6 +62,7 @@ Following resources are checked in VPC command: * VPC Endpoint * VPN Customer Gateways * Virtual Private Gateways +* Workspace The subnets are aggregated to simplify the diagram and hide infrastructure redundancies. There can be two types of subnet aggregates: 1. Private* ones with a route `0.0.0.0/0` to Internet Gateway diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 9ce8e60..1c4b0ba 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -186,7 +186,7 @@ def get_resources(self) -> List[Resource]: ]["InstanceId"] rds = RDS(self.vpc_options).get_resources(instance_id=instance_id) - if len(rds) > 0: + if rds: quicksight_digest = ResourceDigest( id=data["DataSourceId"], type="aws_quicksight" From 711134c7121edddaf57922b3c4c9a1363dd25881 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 19:10:58 +0100 Subject: [PATCH 13/86] Fix code #85 --- .../provider/vpc/resource/enduser.py | 79 +++++++++---------- .../provider/vpc/resource/identity.py | 48 ++++++----- 2 files changed, 59 insertions(+), 68 deletions(-) diff --git a/cloudiscovery/provider/vpc/resource/enduser.py b/cloudiscovery/provider/vpc/resource/enduser.py index fa012de..122864c 100644 --- a/cloudiscovery/provider/vpc/resource/enduser.py +++ b/cloudiscovery/provider/vpc/resource/enduser.py @@ -34,49 +34,44 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Workspaces...", "HEADER") - if len(response["Workspaces"]) > 0: - - for data in response["Workspaces"]: - - # Get tag name - tags = client.describe_tags(ResourceId=data["WorkspaceId"]) - nametag = get_name_tag(tags) - - workspace_name = data["WorkspaceId"] if nametag is None else nametag - - directory_service = self.vpc_options.client("ds") - directories = directory_service.describe_directories( - DirectoryIds=[data["DirectoryId"]] - ) - - for directorie in directories["DirectoryDescriptions"]: - - if "VpcSettings" in directorie: - - if ( - directorie["VpcSettings"]["VpcId"] - == self.vpc_options.vpc_id - ): - workspace_digest = ResourceDigest( - id=data["WorkspaceId"], type="aws_workspaces" + for data in response["Workspaces"]: + + # Get tag name + tags = client.describe_tags(ResourceId=data["WorkspaceId"]) + nametag = get_name_tag(tags) + + workspace_name = data["WorkspaceId"] if nametag is None else nametag + + directory_service = self.vpc_options.client("ds") + directories = directory_service.describe_directories( + DirectoryIds=[data["DirectoryId"]] + ) + + for directorie in directories["DirectoryDescriptions"]: + + if "VpcSettings" in directorie: + + if directorie["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: + workspace_digest = ResourceDigest( + id=data["WorkspaceId"], type="aws_workspaces" + ) + resources_found.append( + Resource( + digest=workspace_digest, + name=workspace_name, + details="", + group="enduser", + tags=resource_tags(tags), ) - resources_found.append( - Resource( - digest=workspace_digest, - name=workspace_name, - details="", - group="enduser", - tags=resource_tags(tags), - ) - ) - - self.relations_found.append( - ResourceEdge( - from_node=workspace_digest, - to_node=ResourceDigest( - id=directorie["DirectoryId"], type="aws_ds" - ), - ) + ) + + self.relations_found.append( + ResourceEdge( + from_node=workspace_digest, + to_node=ResourceDigest( + id=directorie["DirectoryId"], type="aws_ds" + ), ) + ) return resources_found diff --git a/cloudiscovery/provider/vpc/resource/identity.py b/cloudiscovery/provider/vpc/resource/identity.py index 7aa70ce..0e98f61 100644 --- a/cloudiscovery/provider/vpc/resource/identity.py +++ b/cloudiscovery/provider/vpc/resource/identity.py @@ -33,34 +33,30 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Directory Services...", "HEADER") - if len(response["DirectoryDescriptions"]) > 0: - - for data in response["DirectoryDescriptions"]: - - if "VpcSettings" in data: - - if data["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: - directory_service_digest = ResourceDigest( - id=data["DirectoryId"], type="aws_ds" - ) - resources_found.append( - Resource( - digest=directory_service_digest, - name=data["Name"], - details="", - group="identity", - tags=resource_tags(data), - ) + for data in response["DirectoryDescriptions"]: + + if "VpcSettings" in data: + + if data["VpcSettings"]["VpcId"] == self.vpc_options.vpc_id: + directory_service_digest = ResourceDigest( + id=data["DirectoryId"], type="aws_ds" + ) + resources_found.append( + Resource( + digest=directory_service_digest, + name=data["Name"], + details="", + group="identity", + tags=resource_tags(data), ) + ) - for subnet in data["VpcSettings"]["SubnetIds"]: - self.relations_found.append( - ResourceEdge( - from_node=directory_service_digest, - to_node=ResourceDigest( - id=subnet, type="aws_subnet" - ), - ) + for subnet in data["VpcSettings"]["SubnetIds"]: + self.relations_found.append( + ResourceEdge( + from_node=directory_service_digest, + to_node=ResourceDigest(id=subnet, type="aws_subnet"), ) + ) return resources_found From 9996422085274abb7807debb68433118910f6b18 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 19:25:01 +0100 Subject: [PATCH 14/86] Pylint changes and data source name --- .pylintrc | 5 +++++ cloudiscovery/provider/vpc/resource/analytics.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index bab35f2..3057a20 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,3 +6,8 @@ max-line-length=120 [MESSAGES CONTROL] disable=missing-docstring,pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=6 \ No newline at end of file diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 1c4b0ba..1ca9045 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -194,7 +194,7 @@ def get_resources(self) -> List[Resource]: resources_found.append( Resource( digest=quicksight_digest, - name=data["DataSourceId"], + name=data["Name"], details="", group="analytics", tags=resource_tags(data), From 75686c6a5ea445f80f2ebf662af5365d5dfa5451 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 23:08:09 +0100 Subject: [PATCH 15/86] Added check if service is available in region #98 --- .pylintrc | 2 +- README.md | 3 +- cloudiscovery/__init__.py | 3 +- cloudiscovery/provider/vpc/command.py | 6 + .../provider/vpc/resource/analytics.py | 4 + .../provider/vpc/resource/application.py | 2 + .../provider/vpc/resource/compute.py | 6 + .../provider/vpc/resource/containers.py | 178 +++++++++--------- .../provider/vpc/resource/database.py | 5 + .../provider/vpc/resource/enduser.py | 2 + .../provider/vpc/resource/identity.py | 2 + .../provider/vpc/resource/management.py | 2 + .../provider/vpc/resource/mediaservices.py | 4 + cloudiscovery/provider/vpc/resource/ml.py | 4 + .../provider/vpc/resource/network.py | 165 ++++++++-------- .../provider/vpc/resource/security.py | 3 + .../provider/vpc/resource/storage.py | 3 + cloudiscovery/shared/common.py | 51 +++++ cloudiscovery/shared/common_aws.py | 50 ++++- 19 files changed, 318 insertions(+), 177 deletions(-) diff --git a/.pylintrc b/.pylintrc index 3057a20..b090524 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,7 +5,7 @@ init-hook='import sys; sys.path.append("cloudiscovery")' max-line-length=120 [MESSAGES CONTROL] -disable=missing-docstring,pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches +disable=missing-docstring,useless-suppression,pointless-string-statement,locally-disabled,bad-super-call,unnecessary-lambda,missing-class-docstring,arguments-differ,unused-argument,useless-object-inheritance,too-few-public-methods,missing-module-docstring,import-error,eval-used,bad-continuation,invalid-name,missing-function-docstring,no-self-use,no-name-in-module,too-many-lines,attribute-defined-outside-init,fixme,exec-used,expression-not-assigned,too-many-branches [SIMILARITIES] diff --git a/README.md b/README.md index aeaa27b..d5d3bbc 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,8 @@ aws configure "kafka:ListClusters", "synthetics:DescribeCanaries", "medialive:ListInputs", - "cloudhsm:DescribeClusters" + "cloudhsm:DescribeClusters", + "ssm:GetParametersByPath" ], "Resource": [ "*" ] } diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 79a940c..b235df3 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -214,7 +214,8 @@ def check_region(region_parameter, region_name, session): client = session.client("ec2", region_name=DEFAULT_REGION) valid_region_names = [ - region["RegionName"] for region in client.describe_regions()["Regions"] + region["RegionName"] + for region in client.describe_regions(AllRegions=True)["Regions"] ] if region_parameter != "all": diff --git a/cloudiscovery/provider/vpc/command.py b/cloudiscovery/provider/vpc/command.py index 083cf1e..293e0ec 100644 --- a/cloudiscovery/provider/vpc/command.py +++ b/cloudiscovery/provider/vpc/command.py @@ -8,6 +8,7 @@ VPCE_REGEX, SOURCE_IP_ADDRESS_REGEX, ) +from shared.common_aws import GlobalParameters from shared.diagram import NoDiagram, BaseDiagram @@ -65,10 +66,15 @@ def check_vpc(vpc_options: VpcOptions): print(message) def run(self): + # pylint: disable=too-many-branches command_runner = CommandRunner(self.filters) for region in self.region_names: + # Get and cache SSM services available in specific region + path = "/aws/service/global-infrastructure/regions/" + region + "/services/" + GlobalParameters(session=self.session, region=region, path=path).paths() + # if vpc is none, get all vpcs and check if self.vpc_id is None: client = self.session.client("ec2", region_name=region) diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 1ca9045..9c37b07 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -11,6 +11,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -26,6 +27,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="es") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("es") @@ -96,6 +98,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="kafka") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("kafka") @@ -157,6 +160,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="quicksight") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("quicksight") diff --git a/cloudiscovery/provider/vpc/resource/application.py b/cloudiscovery/provider/vpc/resource/application.py index a0e7d54..c2a1770 100644 --- a/cloudiscovery/provider/vpc/resource/application.py +++ b/cloudiscovery/provider/vpc/resource/application.py @@ -11,6 +11,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -26,6 +27,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="sqs") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("sqs") diff --git a/cloudiscovery/provider/vpc/resource/compute.py b/cloudiscovery/provider/vpc/resource/compute.py index 988855e..c321b53 100644 --- a/cloudiscovery/provider/vpc/resource/compute.py +++ b/cloudiscovery/provider/vpc/resource/compute.py @@ -10,6 +10,7 @@ get_name_tag, get_tag, resource_tags, + ResourceAvailable, ) from shared.common_aws import describe_subnet from shared.error_handler import exception @@ -26,6 +27,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="lambda") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("lambda") @@ -77,6 +79,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -143,6 +146,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="eks") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("eks") @@ -195,6 +199,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="emr") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("emr") @@ -254,6 +259,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="autoscaling") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("autoscaling") diff --git a/cloudiscovery/provider/vpc/resource/containers.py b/cloudiscovery/provider/vpc/resource/containers.py index 9612b2f..7983443 100644 --- a/cloudiscovery/provider/vpc/resource/containers.py +++ b/cloudiscovery/provider/vpc/resource/containers.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.common_aws import describe_subnet from shared.error_handler import exception @@ -24,6 +25,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ecs") # pylint: disable=too-many-locals,too-many-branches def get_resources(self) -> List[Resource]: @@ -40,112 +42,106 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from ECS Cluster...", "HEADER") # pylint: disable=too-many-nested-blocks - if len(response["clusters"]) > 0: - - for data in response["clusters"]: - - # Searching all cluster services - paginator = client.get_paginator("list_services") - pages = paginator.paginate(cluster=data["clusterName"]) - - for services in pages: - if len(services["serviceArns"]) > 0: - service_details = client.describe_services( - cluster=data["clusterName"], - services=services["serviceArns"], - ) - - for data_service_detail in service_details["services"]: - if data_service_detail["launchType"] == "FARGATE": - service_subnets = data_service_detail[ - "networkConfiguration" - ]["awsvpcConfiguration"]["subnets"] - - # Using subnet to check VPC - subnets = describe_subnet( - vpc_options=self.vpc_options, - subnet_ids=service_subnets, - ) - - if subnets is not None: - # Iterate subnet to get VPC - for data_subnet in subnets["Subnets"]: - - if ( - data_subnet["VpcId"] - == self.vpc_options.vpc_id - ): - cluster_digest = ResourceDigest( - id=data["clusterArn"], - type="aws_ecs_cluster", - ) - resources_found.append( - Resource( - digest=cluster_digest, - name=data["clusterName"], - details="", - group="container", - tags=resource_tags(data), - ) - ) - self.relations_found.append( - ResourceEdge( - from_node=cluster_digest, - to_node=ResourceDigest( - id=data_subnet["SubnetId"], - type="aws_subnet", - ), - ) - ) - else: - # EC2 services require container instances, list of them should be fine for now - pass - - # Looking for container instances - they are dynamically associated, so manual review is necessary - list_paginator = client.get_paginator("list_container_instances") - list_pages = list_paginator.paginate(cluster=data["clusterName"]) - for list_page in list_pages: - if len(list_page["containerInstanceArns"]) == 0: - continue - - container_instances = client.describe_container_instances( - cluster=data["clusterName"], - containerInstances=list_page["containerInstanceArns"], + for data in response["clusters"]: + + # Searching all cluster services + paginator = client.get_paginator("list_services") + pages = paginator.paginate(cluster=data["clusterName"]) + + for services in pages: + if len(services["serviceArns"]) > 0: + service_details = client.describe_services( + cluster=data["clusterName"], services=services["serviceArns"], ) - ec2_ids = [] - for instance_details in container_instances["containerInstances"]: - ec2_ids.append(instance_details["ec2InstanceId"]) - paginator = ec2_client.get_paginator("describe_instances") - pages = paginator.paginate(InstanceIds=ec2_ids) - for page in pages: - for reservation in page["Reservations"]: - for instance in reservation["Instances"]: - for network_interfaces in instance["NetworkInterfaces"]: - if ( - network_interfaces["VpcId"] - == self.vpc_options.vpc_id - ): - cluster_instance_digest = ResourceDigest( - id=instance["InstanceId"], + + for data_service_detail in service_details["services"]: + if data_service_detail["launchType"] == "FARGATE": + service_subnets = data_service_detail[ + "networkConfiguration" + ]["awsvpcConfiguration"]["subnets"] + + # Using subnet to check VPC + subnets = describe_subnet( + vpc_options=self.vpc_options, + subnet_ids=service_subnets, + ) + + if subnets is not None: + # Iterate subnet to get VPC + for data_subnet in subnets["Subnets"]: + + if data_subnet["VpcId"] == self.vpc_options.vpc_id: + cluster_digest = ResourceDigest( + id=data["clusterArn"], type="aws_ecs_cluster", ) resources_found.append( Resource( - digest=cluster_instance_digest, + digest=cluster_digest, name=data["clusterName"], - details="Instance in EC2 cluster", + details="", group="container", tags=resource_tags(data), ) ) self.relations_found.append( ResourceEdge( - from_node=cluster_instance_digest, + from_node=cluster_digest, to_node=ResourceDigest( - id=instance["InstanceId"], - type="aws_instance", + id=data_subnet["SubnetId"], + type="aws_subnet", ), ) ) + else: + # EC2 services require container instances, list of them should be fine for now + pass + + # Looking for container instances - they are dynamically associated, so manual review is necessary + list_paginator = client.get_paginator("list_container_instances") + list_pages = list_paginator.paginate(cluster=data["clusterName"]) + for list_page in list_pages: + if len(list_page["containerInstanceArns"]) == 0: + continue + + container_instances = client.describe_container_instances( + cluster=data["clusterName"], + containerInstances=list_page["containerInstanceArns"], + ) + ec2_ids = [] + for instance_details in container_instances["containerInstances"]: + ec2_ids.append(instance_details["ec2InstanceId"]) + paginator = ec2_client.get_paginator("describe_instances") + pages = paginator.paginate(InstanceIds=ec2_ids) + for page in pages: + for reservation in page["Reservations"]: + for instance in reservation["Instances"]: + for network_interfaces in instance["NetworkInterfaces"]: + if ( + network_interfaces["VpcId"] + == self.vpc_options.vpc_id + ): + cluster_instance_digest = ResourceDigest( + id=instance["InstanceId"], + type="aws_ecs_cluster", + ) + resources_found.append( + Resource( + digest=cluster_instance_digest, + name=data["clusterName"], + details="Instance in EC2 cluster", + group="container", + tags=resource_tags(data), + ) + ) + self.relations_found.append( + ResourceEdge( + from_node=cluster_instance_digest, + to_node=ResourceDigest( + id=instance["InstanceId"], + type="aws_instance", + ), + ) + ) return resources_found diff --git a/cloudiscovery/provider/vpc/resource/database.py b/cloudiscovery/provider/vpc/resource/database.py index 373d548..afb5b82 100644 --- a/cloudiscovery/provider/vpc/resource/database.py +++ b/cloudiscovery/provider/vpc/resource/database.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="rds") def get_resources(self, instance_id=None) -> List[Resource]: client = self.vpc_options.client("rds") @@ -104,6 +106,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="elasticache") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("elasticache") @@ -163,6 +166,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="docdb") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("docdb") @@ -222,6 +226,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="neptune") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("neptune") diff --git a/cloudiscovery/provider/vpc/resource/enduser.py b/cloudiscovery/provider/vpc/resource/enduser.py index 122864c..c9fdc04 100644 --- a/cloudiscovery/provider/vpc/resource/enduser.py +++ b/cloudiscovery/provider/vpc/resource/enduser.py @@ -9,6 +9,7 @@ ResourceEdge, resource_tags, get_name_tag, + ResourceAvailable, ) from shared.error_handler import exception @@ -24,6 +25,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="workspaces") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("workspaces") diff --git a/cloudiscovery/provider/vpc/resource/identity.py b/cloudiscovery/provider/vpc/resource/identity.py index 0e98f61..a4a57c9 100644 --- a/cloudiscovery/provider/vpc/resource/identity.py +++ b/cloudiscovery/provider/vpc/resource/identity.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ds") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ds") diff --git a/cloudiscovery/provider/vpc/resource/management.py b/cloudiscovery/provider/vpc/resource/management.py index 3dd902f..3a23909 100644 --- a/cloudiscovery/provider/vpc/resource/management.py +++ b/cloudiscovery/provider/vpc/resource/management.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="synthetics") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("synthetics") diff --git a/cloudiscovery/provider/vpc/resource/mediaservices.py b/cloudiscovery/provider/vpc/resource/mediaservices.py index 4038bd6..ab2c47a 100644 --- a/cloudiscovery/provider/vpc/resource/mediaservices.py +++ b/cloudiscovery/provider/vpc/resource/mediaservices.py @@ -10,6 +10,7 @@ ResourceEdge, datetime_to_string, resource_tags, + ResourceAvailable, ) from shared.common_aws import describe_subnet from shared.error_handler import exception @@ -26,6 +27,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="mediaconnect") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("mediaconnect") @@ -91,6 +93,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="medialive") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("medialive") @@ -142,6 +145,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="mediastore") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("mediastore") diff --git a/cloudiscovery/provider/vpc/resource/ml.py b/cloudiscovery/provider/vpc/resource/ml.py index 039d0a3..b46a725 100644 --- a/cloudiscovery/provider/vpc/resource/ml.py +++ b/cloudiscovery/provider/vpc/resource/ml.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.common_aws import describe_subnet from shared.error_handler import exception @@ -24,6 +25,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="sagemaker") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("sagemaker") @@ -87,6 +89,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="sagemaker") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("sagemaker") @@ -153,6 +156,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="sagemaker") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("sagemaker") diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index ca82e99..24a49e2 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -12,6 +12,7 @@ ResourceEdge, datetime_to_string, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -27,6 +28,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -81,6 +83,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -93,39 +96,35 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from NAT Gateways...", "HEADER") - if len(response["NatGateways"]) > 0: + for data in response["NatGateways"]: - for data in response["NatGateways"]: - - if data["VpcId"] == self.vpc_options.vpc_id: - nametag = get_name_tag(data) + if data["VpcId"] == self.vpc_options.vpc_id: + nametag = get_name_tag(data) - name = data["NatGatewayId"] if nametag is None else nametag + name = data["NatGatewayId"] if nametag is None else nametag - nat_digest = ResourceDigest( - id=data["NatGatewayId"], type="aws_nat_gateway" - ) - resources_found.append( - Resource( - digest=nat_digest, - name=name, - details="NAT Gateway Private IP {}, Public IP {}, Subnet id {}".format( - data["NatGatewayAddresses"][0]["PrivateIp"], - data["NatGatewayAddresses"][0]["PublicIp"], - data["SubnetId"], - ), - group="network", - tags=resource_tags(data), - ) + nat_digest = ResourceDigest( + id=data["NatGatewayId"], type="aws_nat_gateway" + ) + resources_found.append( + Resource( + digest=nat_digest, + name=name, + details="NAT Gateway Private IP {}, Public IP {}, Subnet id {}".format( + data["NatGatewayAddresses"][0]["PrivateIp"], + data["NatGatewayAddresses"][0]["PublicIp"], + data["SubnetId"], + ), + group="network", + tags=resource_tags(data), ) - self.relations_found.append( - ResourceEdge( - from_node=nat_digest, - to_node=ResourceDigest( - id=data["SubnetId"], type="aws_subnet" - ), - ) + ) + self.relations_found.append( + ResourceEdge( + from_node=nat_digest, + to_node=ResourceDigest(id=data["SubnetId"], type="aws_subnet"), ) + ) return resources_found @@ -141,6 +140,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="elb") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("elb") @@ -151,32 +151,30 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Classic Load Balancers...", "HEADER") - if len(response["LoadBalancerDescriptions"]) > 0: - - for data in response["LoadBalancerDescriptions"]: - if data["VPCId"] == self.vpc_options.vpc_id: - tags_response = client.describe_tags( - LoadBalancerNames=[data["LoadBalancerName"]] - ) - elb_digest = ResourceDigest( - id=data["LoadBalancerName"], type="aws_elb_classic" - ) - for subnet_id in data["Subnets"]: - self.relations_found.append( - ResourceEdge( - from_node=elb_digest, - to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), - ) - ) - resources_found.append( - Resource( - digest=elb_digest, - name=data["LoadBalancerName"], - details="", - group="network", - tags=resource_tags(tags_response["TagDescriptions"][0]), + for data in response["LoadBalancerDescriptions"]: + if data["VPCId"] == self.vpc_options.vpc_id: + tags_response = client.describe_tags( + LoadBalancerNames=[data["LoadBalancerName"]] + ) + elb_digest = ResourceDigest( + id=data["LoadBalancerName"], type="aws_elb_classic" + ) + for subnet_id in data["Subnets"]: + self.relations_found.append( + ResourceEdge( + from_node=elb_digest, + to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), ) ) + resources_found.append( + Resource( + digest=elb_digest, + name=data["LoadBalancerName"], + details="", + group="network", + tags=resource_tags(tags_response["TagDescriptions"][0]), + ) + ) return resources_found @@ -192,6 +190,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="elb") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("elbv2") @@ -202,39 +201,35 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from Application Load Balancers...", "HEADER") - if len(response["LoadBalancers"]) > 0: + for data in response["LoadBalancers"]: - for data in response["LoadBalancers"]: - - if data["VpcId"] == self.vpc_options.vpc_id: - tags_response = client.describe_tags( - ResourceArns=[data["LoadBalancerArn"]] - ) - elb_digest = ResourceDigest( - id=data["LoadBalancerName"], type="aws_elb" - ) + if data["VpcId"] == self.vpc_options.vpc_id: + tags_response = client.describe_tags( + ResourceArns=[data["LoadBalancerArn"]] + ) + elb_digest = ResourceDigest(id=data["LoadBalancerName"], type="aws_elb") - subnet_ids = [] - for availabilityZone in data["AvailabilityZones"]: - subnet_ids.append(availabilityZone["SubnetId"]) - self.relations_found.append( - ResourceEdge( - from_node=elb_digest, - to_node=ResourceDigest( - id=availabilityZone["SubnetId"], type="aws_subnet" - ), - ) + subnet_ids = [] + for availabilityZone in data["AvailabilityZones"]: + subnet_ids.append(availabilityZone["SubnetId"]) + self.relations_found.append( + ResourceEdge( + from_node=elb_digest, + to_node=ResourceDigest( + id=availabilityZone["SubnetId"], type="aws_subnet" + ), ) + ) - resources_found.append( - Resource( - digest=elb_digest, - name=data["LoadBalancerName"], - details="", - group="network", - tags=resource_tags(tags_response["TagDescriptions"][0]), - ) + resources_found.append( + Resource( + digest=elb_digest, + name=data["LoadBalancerName"], + details="", + group="network", + tags=resource_tags(tags_response["TagDescriptions"][0]), ) + ) return resources_found @@ -250,6 +245,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -326,6 +322,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -375,6 +372,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -430,6 +428,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -472,6 +471,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -533,6 +533,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") vpc_response = client.describe_vpcs(VpcIds=[self.vpc_options.vpc_id]) @@ -556,6 +557,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") @@ -626,6 +628,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="apigateway") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("apigateway") @@ -689,6 +692,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") vpn_response = client.describe_vpn_connections() @@ -801,6 +805,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="ec2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("ec2") client_vpn_endpoints = client.describe_client_vpn_endpoints() diff --git a/cloudiscovery/provider/vpc/resource/security.py b/cloudiscovery/provider/vpc/resource/security.py index ac3e968..134aa8b 100644 --- a/cloudiscovery/provider/vpc/resource/security.py +++ b/cloudiscovery/provider/vpc/resource/security.py @@ -11,6 +11,7 @@ ResourceEdge, datetime_to_string, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -26,6 +27,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="iam") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("iam") @@ -88,6 +90,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="cloudhsmv2") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("cloudhsmv2") diff --git a/cloudiscovery/provider/vpc/resource/storage.py b/cloudiscovery/provider/vpc/resource/storage.py index 0bb5129..4802ed9 100644 --- a/cloudiscovery/provider/vpc/resource/storage.py +++ b/cloudiscovery/provider/vpc/resource/storage.py @@ -14,6 +14,7 @@ datetime_to_string, resource_tags, get_name_tag, + ResourceAvailable, ) from shared.common_aws import describe_subnet from shared.error_handler import exception @@ -30,6 +31,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="efs") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("efs") @@ -95,6 +97,7 @@ def __init__(self, vpc_options: VpcOptions): self.vpc_options = vpc_options @exception + @ResourceAvailable(services="s3") def get_resources(self) -> List[Resource]: client = self.vpc_options.client("s3") diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 8fca3cf..83e2a6a 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -1,7 +1,12 @@ +import os +import os.path import datetime import re +import functools from typing import NamedTuple, List, Optional, Dict +from diskcache import Cache + import boto3 VPCE_REGEX = re.compile(r'(?<=sourcevpce")(\s*:\s*")(vpce-[a-zA-Z0-9]+)', re.DOTALL) @@ -75,6 +80,52 @@ class Resource(NamedTuple): tags: List[ResourceTag] = [] +class ResourceCache: + def __init__(self): + self.cache = Cache( + directory=os.path.dirname(os.path.abspath(__file__)) + + "/../../assets/.cache/" + ) + + def set_key(self, key: str, value: int, expire: int): + self.cache.set(key=key, value=value, expire=expire) + + def get_key(self, key: str): + if key in self.cache: + return self.cache[key] + + return None + + +# Decorator to check services. +class ResourceAvailable(object): + def __init__(self, services): + self.services = services + self.cache = ResourceCache() + + def __call__(self, func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + + region_name = args[0].vpc_options.region_name + cache_key = "aws_paths_" + region_name + cache = self.cache.get_key(cache_key) + + if self.services in cache: + return func(*args, **kwargs) + + message_handler( + "Check " + + func.__qualname__ + + " not available in this region... Skipping", + "WARNING", + ) + + return None + + return wrapper + + def resource_tags(resource_data: dict) -> List[ResourceTag]: if "Tags" in resource_data: tags_input = resource_data["Tags"] diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 7872c46..5ca6954 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -1,13 +1,12 @@ import botocore.exceptions from cachetools import TTLCache -from provider.vpc.command import VpcOptions - +from shared.common import ResourceCache SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) -def describe_subnet(vpc_options: VpcOptions, subnet_ids): +def describe_subnet(vpc_options, subnet_ids): if not isinstance(subnet_ids, list): subnet_ids = [subnet_ids] @@ -21,3 +20,48 @@ def describe_subnet(vpc_options: VpcOptions, subnet_ids): return subnets except botocore.exceptions.ClientError: return None + + +class GlobalParameters: + def __init__(self, session, region: str, path: str): + self.region = region + self.session = session.client("ssm", region_name="us-east-1") + self.path = path + self.cache = ResourceCache() + + def get_parameters_by_path(self, next_token=None): + + params = {"Path": self.path, "Recursive": True, "MaxResults": 10} + if next_token is not None: + params["NextToken"] = next_token + + return self.session.get_parameters_by_path(**params) + + def parameters(self): + next_token = None + while True: + response = self.get_parameters_by_path(next_token) + parameters = response["Parameters"] + if len(parameters) == 0: + break + for parameter in parameters: + yield parameter + if "NextToken" not in response: + break + next_token = response["NextToken"] + + def paths(self): + + cache_key = "aws_paths_" + self.region + cache = self.cache.get_key(cache_key) + + if cache is not None: + return cache + + paths_found = [] + paths = self.parameters() + for path in paths: + paths_found.append(path["Value"]) + + self.cache.set_key(key=cache_key, value=paths_found, expire=86400) + return paths_found From 1938f7c5e55af9d8445b4050f7c244a82667bd71 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 23:09:51 +0100 Subject: [PATCH 16/86] Code performance --- .../provider/vpc/resource/security.py | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/cloudiscovery/provider/vpc/resource/security.py b/cloudiscovery/provider/vpc/resource/security.py index 134aa8b..a754394 100644 --- a/cloudiscovery/provider/vpc/resource/security.py +++ b/cloudiscovery/provider/vpc/resource/security.py @@ -101,31 +101,29 @@ def get_resources(self) -> List[Resource]: message_handler("Collecting data from CloudHSM clusters...", "HEADER") - if len(response["Clusters"]) > 0: + for data in response["Clusters"]: - for data in response["Clusters"]: - - if data["VpcId"] == self.vpc_options.vpc_id: - cloudhsm_digest = ResourceDigest( - id=data["ClusterId"], type="aws_cloudhsm" - ) - resources_found.append( - Resource( - digest=cloudhsm_digest, - name=data["ClusterId"], - details="", - group="security", - tags=resource_tags(data), - ) + if data["VpcId"] == self.vpc_options.vpc_id: + cloudhsm_digest = ResourceDigest( + id=data["ClusterId"], type="aws_cloudhsm" + ) + resources_found.append( + Resource( + digest=cloudhsm_digest, + name=data["ClusterId"], + details="", + group="security", + tags=resource_tags(data), ) + ) - for subnet in data["SubnetMapping"]: - subnet_id = data["SubnetMapping"][subnet] - self.relations_found.append( - ResourceEdge( - from_node=cloudhsm_digest, - to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), - ) + for subnet in data["SubnetMapping"]: + subnet_id = data["SubnetMapping"][subnet] + self.relations_found.append( + ResourceEdge( + from_node=cloudhsm_digest, + to_node=ResourceDigest(id=subnet_id, type="aws_subnet"), ) + ) return resources_found From cfe7cbba4d425afc6a0af52845e3a4241c967bde Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 22 Jun 2020 23:12:54 +0100 Subject: [PATCH 17/86] Build --- requirements.txt | 3 ++- setup.py | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bb44048..f1d57be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ boto3 ipaddress jinja2<3.0 diagrams>=0.14 -cachetools \ No newline at end of file +cachetools +diskcache \ No newline at end of file diff --git a/setup.py b/setup.py index 557bdf5..698180d 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,14 @@ VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""") -requires = ["boto3", "ipaddress", "diagrams>=0.13", "jinja2<3.0", "cachetools"] +requires = [ + "boto3", + "ipaddress", + "diagrams>=0.13", + "jinja2<3.0", + "cachetools", + "diskcache", +] def get_version(): From 78e58f961009a3ba09af93e63c851c57ae21f44e Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 23 Jun 2020 08:56:31 +0100 Subject: [PATCH 18/86] Added resourcecheck in IoT and IAM policy --- cloudiscovery/provider/iot/resource/certificate.py | 2 ++ cloudiscovery/provider/iot/resource/policy.py | 2 ++ cloudiscovery/provider/iot/resource/thing.py | 5 +++++ cloudiscovery/provider/policy/resource/general.py | 2 ++ cloudiscovery/provider/policy/resource/security.py | 4 ++++ cloudiscovery/shared/common.py | 8 +++++++- 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cloudiscovery/provider/iot/resource/certificate.py b/cloudiscovery/provider/iot/resource/certificate.py index ad12101..bb1bc50 100644 --- a/cloudiscovery/provider/iot/resource/certificate.py +++ b/cloudiscovery/provider/iot/resource/certificate.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") diff --git a/cloudiscovery/provider/iot/resource/policy.py b/cloudiscovery/provider/iot/resource/policy.py index b12a5bc..168a9d5 100644 --- a/cloudiscovery/provider/iot/resource/policy.py +++ b/cloudiscovery/provider/iot/resource/policy.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") diff --git a/cloudiscovery/provider/iot/resource/thing.py b/cloudiscovery/provider/iot/resource/thing.py index dddc647..6c7149b 100644 --- a/cloudiscovery/provider/iot/resource/thing.py +++ b/cloudiscovery/provider/iot/resource/thing.py @@ -8,6 +8,7 @@ ResourceDigest, ResourceEdge, resource_tags, + ResourceAvailable, ) from shared.error_handler import exception @@ -23,6 +24,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") @@ -58,6 +60,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") @@ -116,6 +119,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") @@ -177,6 +181,7 @@ def __init__(self, iot_options: IotOptions): self.iot_options = iot_options @exception + @ResourceAvailable(services="iot") def get_resources(self) -> List[Resource]: client = self.iot_options.client("iot") diff --git a/cloudiscovery/provider/policy/resource/general.py b/cloudiscovery/provider/policy/resource/general.py index a01e55d..f4b74de 100644 --- a/cloudiscovery/provider/policy/resource/general.py +++ b/cloudiscovery/provider/policy/resource/general.py @@ -7,11 +7,13 @@ message_handler, ResourceDigest, ResourceEdge, + ResourceAvailable, ) from shared.error_handler import exception class IamUser(ResourceProvider): + @ResourceAvailable(services="iam") def __init__(self, options: BaseAwsOptions): """ Iam user diff --git a/cloudiscovery/provider/policy/resource/security.py b/cloudiscovery/provider/policy/resource/security.py index b605b80..df1d5f4 100644 --- a/cloudiscovery/provider/policy/resource/security.py +++ b/cloudiscovery/provider/policy/resource/security.py @@ -8,6 +8,7 @@ message_handler, ResourceDigest, ResourceEdge, + ResourceAvailable, ) from shared.error_handler import exception @@ -821,6 +822,7 @@ def __init__(self, options: BaseAwsOptions): self.options = options @exception + @ResourceAvailable(services="iam") def get_resources(self) -> List[Resource]: client = self.options.client("iam") message_handler("Collecting data from IAM Policies...", "HEADER") @@ -852,6 +854,7 @@ def build_policy(data): class IamGroup(ResourceProvider): + @ResourceAvailable(services="iam") def __init__(self, options: BaseAwsOptions): """ Iam group @@ -913,6 +916,7 @@ def analyze_relations(self, resource): class IamRole(ResourceProvider): + @ResourceAvailable(services="iam") def __init__(self, options: BaseAwsOptions): """ Iam role diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 83e2a6a..be6e23e 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -107,7 +107,13 @@ def __call__(self, func): @functools.wraps(func) def wrapper(*args, **kwargs): - region_name = args[0].vpc_options.region_name + if "vpc_options" in dir(args[0]): + region_name = args[0].vpc_options.region_name + elif "iot_options" in dir(args[0]): + region_name = args[0].iot_options.region_name + else: + region_name = "us-east-1" + cache_key = "aws_paths_" + region_name cache = self.cache.get_key(cache_key) From dd1acac9665b531d2b5bc5fe841d2f88f85a4269 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 23 Jun 2020 09:04:01 +0100 Subject: [PATCH 19/86] Codacy fix --- cloudiscovery/shared/common_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 5ca6954..86bfaa1 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -42,7 +42,7 @@ def parameters(self): while True: response = self.get_parameters_by_path(next_token) parameters = response["Parameters"] - if len(parameters) == 0: + if not parameters: break for parameter in parameters: yield parameter From 255fe54d42353fae6ccfbb397a02b6ff815aaca6 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 23 Jun 2020 09:18:08 +0100 Subject: [PATCH 20/86] Removed unused import --- cloudiscovery/shared/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index be6e23e..fa303d2 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -1,4 +1,3 @@ -import os import os.path import datetime import re From 32c96d1226f7ef3ea629343c8d5e36f23593c97d Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 22:58:36 +0700 Subject: [PATCH 21/86] #98 run providers in parallel, code review --- cloudiscovery/provider/iot/command.py | 1 + cloudiscovery/provider/policy/command.py | 1 + cloudiscovery/provider/vpc/command.py | 6 +---- cloudiscovery/shared/command.py | 32 +++++++++++++++++------- cloudiscovery/shared/common.py | 7 +++++- cloudiscovery/shared/common_aws.py | 6 ++++- cloudiscovery/shared/diagram.py | 1 + 7 files changed, 38 insertions(+), 16 deletions(-) diff --git a/cloudiscovery/provider/iot/command.py b/cloudiscovery/provider/iot/command.py index f0b3b6f..93467b6 100644 --- a/cloudiscovery/provider/iot/command.py +++ b/cloudiscovery/provider/iot/command.py @@ -42,6 +42,7 @@ def run(self): command_runner = CommandRunner(self.filters) for region_name in self.region_names: + self.init_region_cache(region_name) # if thing_name is none, get all things and check if self.thing_name is None: diff --git a/cloudiscovery/provider/policy/command.py b/cloudiscovery/provider/policy/command.py index 561cb59..baf9884 100644 --- a/cloudiscovery/provider/policy/command.py +++ b/cloudiscovery/provider/policy/command.py @@ -7,6 +7,7 @@ class Policy(BaseCommand): def run(self): for region in self.region_names: + self.init_region_cache(region) options = BaseAwsOptions(session=self.session, region_name=region) command_runner = CommandRunner(self.filters) diff --git a/cloudiscovery/provider/vpc/command.py b/cloudiscovery/provider/vpc/command.py index 293e0ec..ccc1ffd 100644 --- a/cloudiscovery/provider/vpc/command.py +++ b/cloudiscovery/provider/vpc/command.py @@ -8,7 +8,6 @@ VPCE_REGEX, SOURCE_IP_ADDRESS_REGEX, ) -from shared.common_aws import GlobalParameters from shared.diagram import NoDiagram, BaseDiagram @@ -70,10 +69,7 @@ def run(self): command_runner = CommandRunner(self.filters) for region in self.region_names: - - # Get and cache SSM services available in specific region - path = "/aws/service/global-infrastructure/regions/" + region + "/services/" - GlobalParameters(session=self.session, region=region, path=path).paths() + self.init_region_cache(region) # if vpc is none, get all vpcs and check if self.vpc_id is None: diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index d79cea5..213608c 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -1,6 +1,7 @@ import importlib import inspect import os +from concurrent.futures.thread import ThreadPoolExecutor from os.path import dirname from typing import Dict, List @@ -17,6 +18,7 @@ ResourceTag, ResourceType, ) +from shared.common_aws import GlobalParameters from shared.diagram import BaseDiagram from shared.report import Report @@ -36,6 +38,11 @@ def __init__(self, region_names, session, diagram, filters): self.diagram: bool = diagram self.filters: List[Filterable] = filters + def init_region_cache(self, region): + # Get and cache SSM services available in specific region + path = "/aws/service/global-infrastructure/regions/" + region + "/services/" + GlobalParameters(session=self.session, region=region, path=path).paths() + def filter_resources( resources: List[Resource], filters: List[Filterable] @@ -80,6 +87,13 @@ def filter_relations( return filtered_relations +def execute_provider(options, data) -> (List[Resource], List[ResourceEdge]): + provider_instance = data[1](options) + provider_resources = provider_instance.get_resources() + provider_resource_relations = provider_instance.get_relations() + return provider_resources, provider_resource_relations + + class CommandRunner(object): def __init__(self, filters): """ @@ -132,16 +146,16 @@ def run( all_resources: List[Resource] = [] resource_relations: List[ResourceEdge] = [] - for providerTuple in providers: - provider_instance = providerTuple[1](options) - - provider_resources = provider_instance.get_resources() - if provider_resources is not None: - all_resources.extend(provider_resources) + with ThreadPoolExecutor(15) as executor: + provider_results = executor.map( + lambda data: execute_provider(options, data), providers + ) - provider_resource_relations = provider_instance.get_relations() - if provider_resource_relations is not None: - resource_relations.extend(provider_resource_relations) + for provider_results in provider_results: + if provider_results[0] is not None: + all_resources.extend(provider_results[0]) + if provider_results[1] is not None: + resource_relations.extend(provider_results[1]) unique_resources_dict: Dict[ResourceDigest, Resource] = dict() for resource in all_resources: diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index fa303d2..d55ea09 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -2,6 +2,7 @@ import datetime import re import functools +import threading from typing import NamedTuple, List, Optional, Dict from diskcache import Cache @@ -17,6 +18,8 @@ FILTER_TYPE_NAME = "type" FILTER_VALUE_PREFIX = "Value=" +_LOG_SEMAPHORE = threading.Semaphore() + class bcolors: colors = { @@ -240,11 +243,13 @@ def exit_critical(message): def log_critical(message): - print(bcolors.colors.get("FAIL"), message, bcolors.colors.get("ENDC"), sep="") + message_handler(message, "FAIL") def message_handler(message, position): + _LOG_SEMAPHORE.acquire() print(bcolors.colors.get(position), message, bcolors.colors.get("ENDC"), sep="") + _LOG_SEMAPHORE.release() # pylint: disable=inconsistent-return-statements diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 86bfaa1..8da7692 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -1,7 +1,7 @@ import botocore.exceptions from cachetools import TTLCache -from shared.common import ResourceCache +from shared.common import ResourceCache, message_handler SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) @@ -58,6 +58,10 @@ def paths(self): if cache is not None: return cache + message_handler( + "Fetching available resources in region {} to cache...".format(self.region), + "HEADER", + ) paths_found = [] paths = self.parameters() for path in paths: diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 974093a..f59661e 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -290,6 +290,7 @@ def generate_diagram( name=title, filename=PATH_DIAGRAM_OUTPUT + filename, direction="TB", + show=False, graph_attr={"nodesep": "2.0", "ranksep": "1.0", "splines": "curved"}, ) as d: d.dot.engine = self.engine From 80e74f873cd2971221591199e520cc24649132ed Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 23:02:55 +0700 Subject: [PATCH 22/86] #98 add a log with diagram location --- cloudiscovery/shared/diagram.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index f59661e..c535970 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -3,7 +3,7 @@ from diagrams import Diagram, Cluster, Edge -from shared.common import Resource, ResourceEdge, ResourceDigest +from shared.common import Resource, ResourceEdge, ResourceDigest, message_handler from shared.error_handler import exception PATH_DIAGRAM_OUTPUT = "./assets/diagrams/" @@ -286,9 +286,10 @@ def generate_diagram( ordered_resources, initial_resource_relations ) + output_filename = PATH_DIAGRAM_OUTPUT + filename with Diagram( name=title, - filename=PATH_DIAGRAM_OUTPUT + filename, + filename=output_filename, direction="TB", show=False, graph_attr={"nodesep": "2.0", "ranksep": "1.0", "splines": "curved"}, @@ -297,6 +298,9 @@ def generate_diagram( self.draw_diagram(ordered_resources=ordered_resources, relations=relations) + message_handler("\n\nPNG diagram generated", "HEADER") + message_handler("Check your diagram: " + output_filename + ".png", "OKBLUE") + def draw_diagram(self, ordered_resources, relations): already_drawn_elements = {} From b31b478b92bdeeca7ebb6857031d376bc2be2fe6 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 23:07:50 +0700 Subject: [PATCH 23/86] #98 fixed QS Athena --- cloudiscovery/provider/vpc/resource/analytics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 9c37b07..9c225bc 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -177,7 +177,7 @@ def get_resources(self) -> List[Resource]: for data in response["DataSources"]: # Twitter and S3 data source is not supported - if data["Type"] not in ("TWITTER", "S3"): + if data["Type"] not in ("TWITTER", "S3", "ATHENA"): data_source = client.describe_data_source( AwsAccountId=account_id, DataSourceId=data["DataSourceId"] From 1270aef065bb85e4c4b92a95165d5cf81aaa7b14 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 23 Jun 2020 23:28:57 +0700 Subject: [PATCH 24/86] fixed Lambda@Edge principal --- cloudiscovery/provider/policy/resource/security.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/policy/resource/security.py b/cloudiscovery/provider/policy/resource/security.py index df1d5f4..6b1bcfe 100644 --- a/cloudiscovery/provider/policy/resource/security.py +++ b/cloudiscovery/provider/policy/resource/security.py @@ -292,7 +292,7 @@ class Principals: "name": "ECS Application Autoscaling", "group": "network", }, - "edgelambda.lambda.amazonaws.com": { + "edgelambda.amazonaws.com": { "type": "aws_lambda_function", "name": "Lambda@Edge", "group": "compute", From 1e8d871979d8bfd06484935efd9e6cadd00aee2b Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 00:05:16 +0700 Subject: [PATCH 25/86] initial implementation of aws-all resources --- README.md | 22 +- cloudiscovery/__init__.py | 29 ++- cloudiscovery/provider/all/command.py | 29 +++ .../provider/all/resource/__init__.py | 0 cloudiscovery/provider/all/resource/all.py | 200 ++++++++++++++++++ cloudiscovery/shared/common.py | 10 +- .../tests/provider/all/resource/test_all.py | 55 +++++ 7 files changed, 330 insertions(+), 15 deletions(-) create mode 100644 cloudiscovery/provider/all/command.py create mode 100644 cloudiscovery/provider/all/resource/__init__.py create mode 100644 cloudiscovery/provider/all/resource/all.py create mode 100644 cloudiscovery/tests/provider/all/resource/test_all.py diff --git a/README.md b/README.md index d5d3bbc..cdb4df3 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,14 @@ Cloudiscovery helps you to analyze resources in your cloud (AWS/GCP/Azure/Alibab ## Features +### Diagrams + +Commands can generate diagrams. When modelling them, we try to follow the following principle: + +> Graphical excellence is that which gives to the viewer the greatest number of ideas in the shortest time with the least ink in the smallest space. + +Edward Tufte + ### AWS VPC Example of a diagram: @@ -105,6 +113,12 @@ Following resources are checked in IoT command: * IoT Thing * IoT Thing Type +### AWS All + +List all AWS resources (preview) + +The command tries to call all AWS services (200+) and methods with name `Describe`, `Get...` and `List...`. + ## Requirements and Installation ### AWS Resources @@ -205,10 +219,16 @@ cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filt cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] ``` +1.3 To detect all AWS resources: + +```sh +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] +``` + 2. For help use: ```sh -cloudiscovery [aws-vpc|aws-policy|aws-iot] -h +cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all] -h ``` ### Filtering diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index b235df3..2387869 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -22,6 +22,8 @@ import pkg_resources +from provider.all.command import All + """path to pip package""" sys.path.append(dirname(__file__)) @@ -76,13 +78,15 @@ def generate_parser(): ) policy_parser = subparsers.add_parser("aws-policy", help="Analyze policies") - add_default_arguments(policy_parser, is_global=True) + all_parser = subparsers.add_parser("aws-all", help="Analyze all resources") + add_default_arguments(all_parser, diagram_enabled=False) + return parser -def add_default_arguments(parser, is_global=False): +def add_default_arguments(parser, is_global=False, diagram_enabled=True): if not is_global: parser.add_argument( "-r", @@ -106,13 +110,14 @@ def add_default_arguments(parser, is_global=False): "to pass with -f -f approach, values can be separated by : sign; " "example: Name=tags.costCenter;Value=20000:'20001:1'", ) - parser.add_argument( - "-d", - "--diagram", - required=False, - help='print diagram with resources (need Graphviz installed). Use options "True" to ' - 'view image or "False" to save image to disk. Default True', - ) + if diagram_enabled: + parser.add_argument( + "-d", + "--diagram", + required=False, + help='print diagram with resources (need Graphviz installed). Use options "True" to ' + 'view image or "False" to save image to disk. Default True', + ) # pylint: disable=too-many-branches @@ -132,7 +137,9 @@ def main(): language = args.language # Diagram check - if args.diagram is not None and args.diagram not in DIAGRAMS_OPTIONS: + if "diagram" not in args: + diagram = "False" + elif args.diagram is not None and args.diagram not in DIAGRAMS_OPTIONS: diagram = "True" else: diagram = args.diagram @@ -200,6 +207,8 @@ def main(): diagram=diagram, filters=filters, ) + elif args.command == "aws-all": + command = All(region_names=region_names, session=session, filters=filters,) else: raise NotImplementedError("Unknown command") command.run() diff --git a/cloudiscovery/provider/all/command.py b/cloudiscovery/provider/all/command.py new file mode 100644 index 0000000..9797489 --- /dev/null +++ b/cloudiscovery/provider/all/command.py @@ -0,0 +1,29 @@ +from shared.command import BaseCommand, CommandRunner +from shared.common import BaseAwsOptions +from shared.diagram import NoDiagram + + +class All(BaseCommand): + def __init__(self, region_names, session, filters): + """ + All AWS resources + + :param region_names: + :param session: + :param filters: + """ + super().__init__(region_names, session, False, filters) + + def run(self): + for region in self.region_names: + self.init_region_cache(region) + options = BaseAwsOptions(session=self.session, region_name=region) + + command_runner = CommandRunner(self.filters) + command_runner.run( + provider="all", + options=options, + diagram_builder=NoDiagram(), + title="AWS Resources - Region {}".format(region), + filename=options.resulting_file_name("all"), + ) diff --git a/cloudiscovery/provider/all/resource/__init__.py b/cloudiscovery/provider/all/resource/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py new file mode 100644 index 0000000..7386652 --- /dev/null +++ b/cloudiscovery/provider/all/resource/all.py @@ -0,0 +1,200 @@ +import re +from functools import reduce +from typing import List + +from botocore.loaders import Loader + +from shared.common import ( + ResourceProvider, + Resource, + BaseAwsOptions, + ResourceDigest, + message_handler, + ResourceAvailable, +) +from shared.error_handler import exception + +OMITTED_RESOURCES = [ + "aws_ec2_reserved_instances_offering", + "aws_ec2_snapshot", + "aws_ec2_spot_price_history", + "aws_ssm_available_patche", + "aws_polly_voice", + "aws_lightsail_blueprint", + "aws_elastictranscoder_preset", + "aws_ec2_vpc_endpoint_service", + "aws_dms_endpoint_type", + "aws_elasticache_service_update", + "aws_rds_source_region", + "aws_ssm_association", + "aws_ssm_patch_baseline", +] + + +def _to_snake_case(function): + return reduce(lambda x, y: x + ("_" if y.isupper() else "") + y, function).lower() + + +def last_singular_name_element(operation_name): + last_name = re.findall("[A-Z][^A-Z]*", operation_name)[-1] + if last_name.endswith("s"): + last_name = last_name[:-1] + return last_name + + +def retrieve_resource_name(resource, operation_name): + resource_name = None + last_name = last_singular_name_element(operation_name) + if "name" in resource: + resource_name = resource["name"] + elif "Name" in resource: + resource_name = resource["Name"] + elif last_name + "Name" in resource: + resource_name = resource[last_name + "Name"] + elif only_one_suffix(resource, "name"): + resource_name = only_one_suffix(resource, "name") + + return resource_name + + +# pylint: disable=inconsistent-return-statements +def only_one_suffix(resource, suffix): + id_keys = [] + last_id_val = None + for key, val in resource.items(): + if key.lower().endswith(suffix) and not key.lower().endswith( + "display" + suffix + ): + id_keys.append(key) + last_id_val = val + if len(id_keys) == 1: + return last_id_val + return None + + +def retrieve_resource_id(resource, operation_name, resource_name): + resource_id = resource_name + last_name = last_singular_name_element(operation_name) + if "id" in resource: + resource_id = resource["id"] + elif last_name + "Id" in resource: + resource_id = resource[last_name + "Id"] + elif only_one_suffix(resource, "id"): + resource_id = only_one_suffix(resource, "id") + elif "arn" in resource: + resource_id = resource["arn"] + elif last_name + "Arn" in resource: + resource_id = resource[last_name + "Arn"] + elif only_one_suffix(resource, "arn"): + resource_id = only_one_suffix(resource, "arn") + # type 'aws_ec2_dhcp_option' + # 'DhcpOptionsId' -> 'dopt-042d18a4769f7b35b' + # also got 'OwnerId' + + return resource_id + + +class AllResources(ResourceProvider): + def __init__(self, options: BaseAwsOptions): + """ + All resources + + :param options: + """ + super().__init__() + self.options = options + self.availabilityCheck = ResourceAvailable("") + + @exception + def get_resources(self) -> List[Resource]: + boto_loader = Loader() + aws_services = boto_loader.list_available_services(type_name="service-2") + resources = [] + + for aws_service in aws_services: + service_resources = self.analyze_service(aws_service, boto_loader) + if service_resources is not None: + resources.extend(service_resources) + + return resources + + @exception + def analyze_service(self, aws_service, boto_loader): + resources = [] + client = self.options.client(aws_service) + service_model = boto_loader.load_service_model(aws_service, "service-2") + paginators_model = boto_loader.load_service_model(aws_service, "paginators-1") + service_full_name = service_model["metadata"]["serviceFullName"] + message_handler( + "Collecting data from {}...".format(service_full_name), "HEADER" + ) + if not self.availabilityCheck.is_service_available( + self.options.region_name, aws_service + ): + message_handler( + "Service {} not available in this region... Skipping".format( + service_full_name + ), + "WARNING", + ) + return None + for name, operation in service_model["operations"].items(): + if ( + name.startswith("List") + or name.startswith("Get") + or name.startswith("Describe") + ): + has_paginator = name in paginators_model["pagination"] + input_model = service_model["shapes"][operation["input"]["shape"]] + if "required" in input_model and len(input_model["required"]) > 0: + continue + resource_type = ( + "aws_" + + aws_service + + "_" + + _to_snake_case( + name.replace("List", "") + .replace("Get", "") + .replace("Describe", "") + ) + ) + if resource_type.endswith("s"): + resource_type = resource_type[:-1] + if resource_type in OMITTED_RESOURCES: + continue + analyze_operation = self.analyze_operation( + resource_type, name, has_paginator, client + ) + if analyze_operation is not None: + resources.extend(analyze_operation) + return resources + + @exception + def analyze_operation( + self, resource_type, operation_name, has_paginator, client + ) -> List[Resource]: + resources = [] + if has_paginator: + paginator = client.get_paginator(_to_snake_case(operation_name)) + pages = paginator.paginate() + result_key = pages.result_keys[0].parsed["value"] + for page in pages: + for resource in page[result_key]: + if isinstance(resource, str): + continue + + resource_name = retrieve_resource_name(resource, operation_name) + resource_id = retrieve_resource_id( + resource, operation_name, resource_name + ) + + if resource_id is None or resource_name is None: + continue + + resources.append( + Resource( + digest=ResourceDigest(id=resource_id, type=resource_type), + name=resource_name, + ) + ) + return resources diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index d55ea09..c0f9407 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -105,6 +105,11 @@ def __init__(self, services): self.services = services self.cache = ResourceCache() + def is_service_available(self, region_name, service_name) -> bool: + cache_key = "aws_paths_" + region_name + cache = self.cache.get_key(cache_key) + return service_name in cache + def __call__(self, func): @functools.wraps(func) def wrapper(*args, **kwargs): @@ -116,10 +121,7 @@ def wrapper(*args, **kwargs): else: region_name = "us-east-1" - cache_key = "aws_paths_" + region_name - cache = self.cache.get_key(cache_key) - - if self.services in cache: + if self.is_service_available(region_name, self.services): return func(*args, **kwargs) message_handler( diff --git a/cloudiscovery/tests/provider/all/resource/test_all.py b/cloudiscovery/tests/provider/all/resource/test_all.py new file mode 100644 index 0000000..c4ef28d --- /dev/null +++ b/cloudiscovery/tests/provider/all/resource/test_all.py @@ -0,0 +1,55 @@ +from unittest import TestCase + +from assertpy import assert_that + +from provider.all.resource.all import ( + retrieve_resource_name, + retrieve_resource_id, + last_singular_name_element, +) + + +class TestAllDiagram(TestCase): + def test_last_singular_name_element(self): + assert_that(last_singular_name_element("ListValues")).is_equal_to("Value") + assert_that(last_singular_name_element("DescribeSomeValues")).is_equal_to( + "Value" + ) + + def test_retrieve_resource_name(self): + assert_that( + retrieve_resource_name({"name": "value"}, "ListValues") + ).is_equal_to("value") + + assert_that( + retrieve_resource_name({"ValueName": "value"}, "ListValues") + ).is_equal_to("value") + assert_that( + retrieve_resource_name({"SomeName": "value"}, "ListValues") + ).is_equal_to("value") + + def test_retrieve_resource_id(self): + assert_that( + retrieve_resource_id({"id": "123"}, "ListValues", "value") + ).is_equal_to("123") + + assert_that( + retrieve_resource_id({"arn": "123"}, "ListValues", "value") + ).is_equal_to("123") + + assert_that( + retrieve_resource_id({"ValueName": "value"}, "ListValues", "value") + ).is_equal_to("value") + + assert_that( + retrieve_resource_id({"ValueId": "123"}, "ListValues", "value") + ).is_equal_to("123") + assert_that( + retrieve_resource_id({"ValueArn": "123"}, "ListValues", "value") + ).is_equal_to("123") + assert_that( + retrieve_resource_id({"someId": "123"}, "ListValues", "value") + ).is_equal_to("123") + assert_that( + retrieve_resource_id({"someArn": "123"}, "ListValues", "value") + ).is_equal_to("123") From acc9765bda083ffac1647a5c7fe3bca0e3d483fe Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 00:15:47 +0700 Subject: [PATCH 26/86] code quality --- cloudiscovery/provider/all/resource/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 7386652..8696019 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -146,7 +146,7 @@ def analyze_service(self, aws_service, boto_loader): ): has_paginator = name in paginators_model["pagination"] input_model = service_model["shapes"][operation["input"]["shape"]] - if "required" in input_model and len(input_model["required"]) > 0: + if "required" in input_model and input_model["required"]: continue resource_type = ( "aws_" From 397ee7cae59b45e4d384f81e51b148b1bdaba629 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 00:32:51 +0700 Subject: [PATCH 27/86] code quality --- cloudiscovery/provider/all/command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/provider/all/command.py b/cloudiscovery/provider/all/command.py index 9797489..fddf360 100644 --- a/cloudiscovery/provider/all/command.py +++ b/cloudiscovery/provider/all/command.py @@ -25,5 +25,6 @@ def run(self): options=options, diagram_builder=NoDiagram(), title="AWS Resources - Region {}".format(region), + # pylint: disable=no-member filename=options.resulting_file_name("all"), ) From 27c6c6db88bea0c2ce07bc8def46c0ecaf09ea41 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 27 Jun 2020 09:14:17 +0100 Subject: [PATCH 28/86] Html report issue --- cloudiscovery/templates/report_html.html | 40 +++++++++++++----------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/cloudiscovery/templates/report_html.html b/cloudiscovery/templates/report_html.html index f1cdf0d..d1a808b 100644 --- a/cloudiscovery/templates/report_html.html +++ b/cloudiscovery/templates/report_html.html @@ -27,25 +27,27 @@ {%- endfor %} -

Found relations

- - - - - - - - -{% for resource_relations in resources_relations %} - - - - - - -{%- endfor %} - -
From typeFrom idTo typeTo id
{{ resource_relations.from_node.type}}{{ resource_relations.from_node.id}}{{ resource_relations.to_node.type}}{{ resource_relations.to_node.id}}
+{% if resources_relations|length > 0 %} +

Found relations

+ + + + + + + + + {% for resource_relations in resources_relations %} + + + + + + + {%- endfor %} + +
From typeFrom idTo typeTo id
{{ resource_relations.from_node.type}}{{ resource_relations.from_node.id}}{{ resource_relations.to_node.type}}{{ resource_relations.to_node.id}}
+{%endif %} {% if diagram_image is not none %}

Diagram

{% set base64img = "data:image/png;base64," + diagram_image %} From b6f9269f0a5ea8766795b036d2b9cdd03c3dea7d Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 15:24:04 +0700 Subject: [PATCH 29/86] local evaluation logic for listing all resources, excluding default AWS resources --- cloudiscovery/provider/all/resource/all.py | 88 +++++++++++++++++-- .../tests/provider/all/resource/test_all.py | 9 ++ 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 8696019..00a16f9 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -15,21 +15,34 @@ from shared.error_handler import exception OMITTED_RESOURCES = [ + "aws_dax_default_parameter", + "aws_dax_parameter_group", "aws_ec2_reserved_instances_offering", "aws_ec2_snapshot", "aws_ec2_spot_price_history", "aws_ssm_available_patche", + "aws_ssm_document", "aws_polly_voice", "aws_lightsail_blueprint", + "aws_lightsail_bundle", "aws_elastictranscoder_preset", "aws_ec2_vpc_endpoint_service", "aws_dms_endpoint_type", "aws_elasticache_service_update", + "aws_elasticache_cache_parameter_group", "aws_rds_source_region", "aws_ssm_association", "aws_ssm_patch_baseline", ] +ON_TOP_POLICIES = [ + "kafka:ListClusters", + "synthetics:DescribeCanaries", + "medialive:ListInputs", + "cloudhsm:DescribeClusters", + "ssm:GetParametersByPath", +] + def _to_snake_case(function): return reduce(lambda x, y: x + ("_" if y.isupper() else "") + y, function).lower() @@ -94,6 +107,29 @@ def retrieve_resource_id(resource, operation_name, resource_name): return resource_id +def operation_allowed( + allowed_actions: List[str], aws_service: str, operation_name: str +): + evaluation_result = False + for action in allowed_actions: + if action == "*": + evaluation_result = True + break + action_service = action.split(":", 1)[0] + if not action_service == aws_service: + continue + action_operation = action.split(":", 1)[1] + if action_operation.endswith("*") and operation_name.startswith( + action_operation[:-1] + ): + evaluation_result = True + break + if operation_name == action_operation: + evaluation_result = True + break + return evaluation_result + + class AllResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): """ @@ -110,16 +146,19 @@ def get_resources(self) -> List[Resource]: boto_loader = Loader() aws_services = boto_loader.list_available_services(type_name="service-2") resources = [] + allowed_actions = self.get_policies_allowed_actions() for aws_service in aws_services: - service_resources = self.analyze_service(aws_service, boto_loader) + service_resources = self.analyze_service( + aws_service, boto_loader, allowed_actions + ) if service_resources is not None: resources.extend(service_resources) return resources @exception - def analyze_service(self, aws_service, boto_loader): + def analyze_service(self, aws_service, boto_loader, allowed_actions): resources = [] client = self.options.client(aws_service) service_model = boto_loader.load_service_model(aws_service, "service-2") @@ -148,20 +187,20 @@ def analyze_service(self, aws_service, boto_loader): input_model = service_model["shapes"][operation["input"]["shape"]] if "required" in input_model and input_model["required"]: continue - resource_type = ( - "aws_" - + aws_service - + "_" - + _to_snake_case( + resource_type = "aws_{}_{}".format( + aws_service, + _to_snake_case( name.replace("List", "") .replace("Get", "") .replace("Describe", "") - ) + ), ) if resource_type.endswith("s"): resource_type = resource_type[:-1] if resource_type in OMITTED_RESOURCES: continue + if not operation_allowed(allowed_actions, aws_service, name): + continue analyze_operation = self.analyze_operation( resource_type, name, has_paginator, client ) @@ -198,3 +237,36 @@ def analyze_operation( ) ) return resources + + def get_policies_allowed_actions(self): + message_handler("Fetching allowed actions...", "HEADER") + iam_client = self.options.client("iam") + view_only_document = self.get_policy_allowed_calls( + iam_client, "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess" + ) + sec_audit_document = self.get_policy_allowed_calls( + iam_client, "arn:aws:iam::aws:policy/SecurityAudit" + ) + + allowed_actions = {} + for action in view_only_document["Statement"][0]["Action"]: + allowed_actions[action] = True + for action in sec_audit_document["Statement"][0]["Action"]: + allowed_actions[action] = True + for action in ON_TOP_POLICIES: + allowed_actions[action] = True + message_handler( + "Found {} allowed actions".format(len(allowed_actions)), "HEADER" + ) + + return allowed_actions.keys() + + def get_policy_allowed_calls(self, iam_client, policy_arn): + policy_version_id = iam_client.get_policy(PolicyArn=policy_arn)["Policy"][ + "DefaultVersionId" + ] + policy_document = iam_client.get_policy_version( + PolicyArn=policy_arn, VersionId=policy_version_id + )["PolicyVersion"]["Document"] + + return policy_document diff --git a/cloudiscovery/tests/provider/all/resource/test_all.py b/cloudiscovery/tests/provider/all/resource/test_all.py index c4ef28d..0eed4a2 100644 --- a/cloudiscovery/tests/provider/all/resource/test_all.py +++ b/cloudiscovery/tests/provider/all/resource/test_all.py @@ -6,6 +6,7 @@ retrieve_resource_name, retrieve_resource_id, last_singular_name_element, + operation_allowed, ) @@ -53,3 +54,11 @@ def test_retrieve_resource_id(self): assert_that( retrieve_resource_id({"someArn": "123"}, "ListValues", "value") ).is_equal_to("123") + + def test_operation_allowed(self): + assert_that(operation_allowed(["iam:List*"], "iam", "ListRoles")).is_equal_to( + True + ) + assert_that(operation_allowed(["ecs:List*"], "iam", "ListRoles")).is_equal_to( + False + ) From 19e7bebd7c35d2c89fe099401c6283aa96a553aa Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 15:29:43 +0700 Subject: [PATCH 30/86] listing all AWS resources in parallel --- cloudiscovery/provider/all/resource/all.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 00a16f9..24b9128 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -1,4 +1,5 @@ import re +from concurrent.futures.thread import ThreadPoolExecutor from functools import reduce from typing import List @@ -148,10 +149,14 @@ def get_resources(self) -> List[Resource]: resources = [] allowed_actions = self.get_policies_allowed_actions() - for aws_service in aws_services: - service_resources = self.analyze_service( - aws_service, boto_loader, allowed_actions + with ThreadPoolExecutor(80) as executor: + results = executor.map( + lambda aws_service: self.analyze_service( + aws_service, boto_loader, allowed_actions + ), + aws_services, ) + for service_resources in results: if service_resources is not None: resources.extend(service_resources) From 56b41d7c07ad84883b6545a429b0b7f700c9184e Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 15:31:48 +0700 Subject: [PATCH 31/86] removed one default resource --- cloudiscovery/provider/all/resource/all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 24b9128..105a0b6 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -34,6 +34,7 @@ "aws_rds_source_region", "aws_ssm_association", "aws_ssm_patch_baseline", + "aws_ec2_prefix", ] ON_TOP_POLICIES = [ From 98a5b08eebb4d2572a43794c8af4795708479aa5 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 17:46:56 +0700 Subject: [PATCH 32/86] fixed CloudFront listing operations --- cloudiscovery/provider/all/resource/all.py | 40 +++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 105a0b6..5599ff1 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -190,9 +190,10 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): or name.startswith("Describe") ): has_paginator = name in paginators_model["pagination"] - input_model = service_model["shapes"][operation["input"]["shape"]] - if "required" in input_model and input_model["required"]: - continue + if "input" in operation: + input_model = service_model["shapes"][operation["input"]["shape"]] + if "required" in input_model and input_model["required"]: + continue resource_type = "aws_{}_{}".format( aws_service, _to_snake_case( @@ -215,6 +216,7 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): return resources @exception + # pylint: disable=too-many-locals def analyze_operation( self, resource_type, operation_name, has_paginator, client ) -> List[Resource]: @@ -222,15 +224,37 @@ def analyze_operation( if has_paginator: paginator = client.get_paginator(_to_snake_case(operation_name)) pages = paginator.paginate() - result_key = pages.result_keys[0].parsed["value"] + list_metadata = pages.result_keys[0].parsed + result_key = None + result_parent = None + result_child = None + if "value" in list_metadata: + result_key = list_metadata["value"] + elif "type" in list_metadata and list_metadata["type"] == "subexpression": + result_parent = list_metadata["children"][0]["value"] + result_child = list_metadata["children"][1]["value"] + else: + message_handler( + "Operation {} has unsupported pagination definition... Skipping".format( + operation_name + ), + "WARNING", + ) + return [] for page in pages: - for resource in page[result_key]: - if isinstance(resource, str): + if result_key is not None: + page_resources = page[result_key] + else: + page_resources = page[result_parent][result_child] + for page_resource in page_resources: + if isinstance(page_resource, str): continue - resource_name = retrieve_resource_name(resource, operation_name) + resource_name = retrieve_resource_name( + page_resource, operation_name + ) resource_id = retrieve_resource_id( - resource, operation_name, resource_name + page_resource, operation_name, resource_name ) if resource_id is None or resource_name is None: From f41ea17f76e68ad13cf876bdd1a0161f8d1f5f37 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 17:56:15 +0700 Subject: [PATCH 33/86] added missing __init__ --- cloudiscovery/provider/all/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cloudiscovery/provider/all/__init__.py diff --git a/cloudiscovery/provider/all/__init__.py b/cloudiscovery/provider/all/__init__.py new file mode 100644 index 0000000..e69de29 From 5afe70759818f613c9b8893158f76d0e2ec05a60 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sat, 27 Jun 2020 16:59:25 +0100 Subject: [PATCH 34/86] initial implementation of aws-limits values --- cloudiscovery/__init__.py | 57 +++++++++---- cloudiscovery/provider/all/__init__.py | 0 cloudiscovery/provider/limits/__init__.py | 0 cloudiscovery/provider/limits/command.py | 34 ++++++++ .../provider/limits/resource/__init__.py | 0 cloudiscovery/provider/limits/resource/all.py | 79 +++++++++++++++++++ cloudiscovery/shared/command.py | 13 ++- cloudiscovery/shared/common.py | 1 + cloudiscovery/shared/common_aws.py | 63 +++++++++++++++ cloudiscovery/templates/report_html.html | 2 +- 10 files changed, 228 insertions(+), 21 deletions(-) create mode 100644 cloudiscovery/provider/all/__init__.py create mode 100644 cloudiscovery/provider/limits/__init__.py create mode 100644 cloudiscovery/provider/limits/command.py create mode 100644 cloudiscovery/provider/limits/resource/__init__.py create mode 100644 cloudiscovery/provider/limits/resource/all.py diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 2387869..d923011 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -22,8 +22,6 @@ import pkg_resources -from provider.all.command import All - """path to pip package""" sys.path.append(dirname(__file__)) @@ -32,6 +30,8 @@ from provider.policy.command import Policy from provider.vpc.command import Vpc from provider.iot.command import Iot +from provider.all.command import All +from provider.limits.command import Limits # Check version from shared.common import ( @@ -83,10 +83,24 @@ def generate_parser(): all_parser = subparsers.add_parser("aws-all", help="Analyze all resources") add_default_arguments(all_parser, diagram_enabled=False) + limit_parser = subparsers.add_parser( + "aws-limits", help="Analyze aws limit resources." + ) + add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False) + limit_parser.add_argument( + "-s", + "--services", + required=False, + help='Inform services that you want to check, use "," (comma) to split them. \ + If not informed, script will check all services.', + ) + return parser -def add_default_arguments(parser, is_global=False, diagram_enabled=True): +def add_default_arguments( + parser, is_global=False, diagram_enabled=True, filters_enabled=True +): if not is_global: parser.add_argument( "-r", @@ -101,15 +115,16 @@ def add_default_arguments(parser, is_global=False, diagram_enabled=True): parser.add_argument( "-l", "--language", required=False, help="available languages: pt_BR, en_US" ) - parser.add_argument( - "-f", - "--filters", - action="append", - required=False, - help="filter resources (tags only for now, you must specify name and values); multiple filters are possible " - "to pass with -f -f approach, values can be separated by : sign; " - "example: Name=tags.costCenter;Value=20000:'20001:1'", - ) + if filters_enabled: + parser.add_argument( + "-f", + "--filters", + action="append", + required=False, + help="filter resources (tags only for now, you must specify name and values); multiple filters " + "are possible to pass with -f -f approach, values can be separated by : sign; " + "example: Name=tags.costCenter;Value=20000:'20001:1'", + ) if diagram_enabled: parser.add_argument( "-d", @@ -163,9 +178,10 @@ def main(): ) # filters check - filters: List[Filterable] = [] - if args.filters is not None: - filters = parse_filters(args.filters) + if "filters" in args: + filters: List[Filterable] = [] + if args.filters is not None: + filters = parse_filters(args.filters) # aws profile check session = generate_session(args.profile_name) @@ -184,7 +200,7 @@ def main(): # get regions region_names = check_region( - region_parameter=args.region_name, region_name=region_name, session=session + region_parameter=args.region_name, region_name=region_name, session=session, ) if args.command == "aws-vpc": @@ -197,7 +213,10 @@ def main(): ) elif args.command == "aws-policy": command = Policy( - region_names=region_names, session=session, diagram=diagram, filters=filters + region_names=region_names, + session=session, + diagram=diagram, + filters=filters, ) elif args.command == "aws-iot": command = Iot( @@ -209,6 +228,10 @@ def main(): ) elif args.command == "aws-all": command = All(region_names=region_names, session=session, filters=filters,) + elif args.command == "aws-limits": + command = Limits( + region_names=region_names, session=session, services=args.services, + ) else: raise NotImplementedError("Unknown command") command.run() diff --git a/cloudiscovery/provider/all/__init__.py b/cloudiscovery/provider/all/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/limits/__init__.py b/cloudiscovery/provider/limits/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/limits/command.py b/cloudiscovery/provider/limits/command.py new file mode 100644 index 0000000..81a8df4 --- /dev/null +++ b/cloudiscovery/provider/limits/command.py @@ -0,0 +1,34 @@ +from shared.command import BaseCommand, CommandRunner +from shared.common import BaseAwsOptions +from shared.diagram import NoDiagram + + +class Limits(BaseCommand): + def __init__(self, region_names, session, services): + """ + All AWS resources + + :param region_names: + :param session: + :param services: + """ + super().__init__(region_names, session, False, False) + self.services = services + + def run(self): + + for region in self.region_names: + self.init_globalaws_limits_cache(region=region, services=self.services) + options = BaseAwsOptions( + session=self.session, region_name=region, services=self.services + ) + + command_runner = CommandRunner(services=self.services) + command_runner.run( + provider="limits", + options=options, + diagram_builder=NoDiagram(), + title="AWS Limits - Region {}".format(region), + # pylint: disable=no-member + filename="AWS Limits - Region {}".format(region), + ) diff --git a/cloudiscovery/provider/limits/resource/__init__.py b/cloudiscovery/provider/limits/resource/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py new file mode 100644 index 0000000..526f3e7 --- /dev/null +++ b/cloudiscovery/provider/limits/resource/all.py @@ -0,0 +1,79 @@ +from typing import List + +from shared.common_aws import ALLOWED_SERVICES_CODES +from shared.common import ( + ResourceProvider, + Resource, + BaseAwsOptions, + ResourceDigest, + message_handler, + ResourceCache, +) +from shared.error_handler import exception + + +class LimitResources(ResourceProvider): + def __init__(self, options: BaseAwsOptions): + """ + All resources + + :param options: + """ + super().__init__() + self.options = options + self.cache = ResourceCache() + + @exception + def get_resources(self) -> List[Resource]: + + client_quota = self.options.session.client("service-quotas") + + resources_found = [] + + services = self.options.services.split(",") + for service in services: + cache_key = "aws_limits_" + service + "_" + self.options.region_name + cache = self.cache.get_key(cache_key) + + for data_quota_code in cache[service]: + quota_data = ALLOWED_SERVICES_CODES[service][ + data_quota_code["quota_code"] + ] + + # Quota is adjustable by ticket request, then must override this values + if bool(data_quota_code["adjustable"]) is True: + response_quota = client_quota.get_service_quota( + ServiceCode=service, QuotaCode=data_quota_code["quota_code"] + ) + if "Value" in response_quota["Quota"]: + value = response_quota["Quota"]["Value"] + else: + value = data_quota_code["value"] + + message_handler( + "Collecting data from Quota: " + + data_quota_code["quota_name"] + + "...", + "HEADER", + ) + client = self.options.session.client(service) + + response = getattr(client, quota_data["method"])() + + total = str(len(response[quota_data["key"]])) + + resources_found.append( + Resource( + digest=ResourceDigest( + id=data_quota_code["quota_code"], type="aws_limit" + ), + name=data_quota_code["quota_name"], + group=service, + details="Limit: " + + str(int(value)) + + "... Current usage: " + + total, + ) + ) + + return resources_found diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index 213608c..58f0f4f 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -18,7 +18,7 @@ ResourceTag, ResourceType, ) -from shared.common_aws import GlobalParameters +from shared.common_aws import GlobalParameters, LimitParameters from shared.diagram import BaseDiagram from shared.report import Report @@ -43,11 +43,17 @@ def init_region_cache(self, region): path = "/aws/service/global-infrastructure/regions/" + region + "/services/" GlobalParameters(session=self.session, region=region, path=path).paths() + def init_globalaws_limits_cache(self, region, services): + # Cache services global and local services + LimitParameters( + session=self.session, region=region, services=services + ).init_globalaws_limits_cache() + def filter_resources( resources: List[Resource], filters: List[Filterable] ) -> List[Resource]: - if len(filters) == 0: + if not filters: return resources filtered_resources = [] @@ -95,13 +101,14 @@ def execute_provider(options, data) -> (List[Resource], List[ResourceEdge]): class CommandRunner(object): - def __init__(self, filters): + def __init__(self, filters=None, services=None): """ Base class command execution :param filters: """ self.filters: List[Filterable] = filters + self.services = services # pylint: disable=too-many-locals,too-many-arguments def run( diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index c0f9407..1e4c63a 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -37,6 +37,7 @@ class bcolors: class BaseAwsOptions(NamedTuple): session: boto3.Session region_name: str + services: str def client(self, service_name: str): return self.session.client(service_name, region_name=self.region_name) diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 8da7692..7b9d167 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -4,6 +4,18 @@ from shared.common import ResourceCache, message_handler SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) +ALLOWED_SERVICES_CODES = { + "ec2": { + "L-0263D0A3": { + "method": "describe_addresses", + "key": "Addresses", + "fields": ["PublicIp"], + } + }, + "cloudformation": { + "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []} + }, +} def describe_subnet(vpc_options, subnet_ids): @@ -22,6 +34,57 @@ def describe_subnet(vpc_options, subnet_ids): return None +class LimitParameters: + def __init__(self, session, region: str, services): + self.region = region + self.session = session.client("service-quotas", region_name=region) + self.cache = ResourceCache() + self.services = services.split(",") + + def init_globalaws_limits_cache(self): + """ + AWS has global limits that can be adjustable and others that can't be adjustable + This method make cache for 15 days for aws cache global parameters. AWS don't update limits every time. + Services has differents limits, depending on region. + """ + for service_code in self.services: + if service_code in ALLOWED_SERVICES_CODES: + cache_key = "aws_limits_" + service_code + "_" + self.region + + cache = self.cache.get_key(cache_key) + if cache is not None: + continue + + message_handler( + "Fetching aws global limits to service {} in region {} to cache...".format( + service_code, self.region + ), + "HEADER", + ) + + cache_codes = dict() + for quota_code in ALLOWED_SERVICES_CODES[service_code]: + response = self.session.get_aws_default_service_quota( + ServiceCode=service_code, QuotaCode=quota_code + ) + + item_to_add = { + "valye": response["Quota"]["Value"], + "adjustable": response["Quota"]["Adjustable"], + "quota_code": quota_code, + "quota_name": response["Quota"]["QuotaName"], + } + + if service_code in cache_codes: + cache_codes[service_code].append(item_to_add) + else: + cache_codes[service_code] = [item_to_add] + + self.cache.set_key(key=cache_key, value=cache_codes, expire=1296000) + + return True + + class GlobalParameters: def __init__(self, session, region: str, path: str): self.region = region diff --git a/cloudiscovery/templates/report_html.html b/cloudiscovery/templates/report_html.html index d1a808b..ed2cdec 100644 --- a/cloudiscovery/templates/report_html.html +++ b/cloudiscovery/templates/report_html.html @@ -17,7 +17,7 @@ {{ resource_found.group}} {{ resource_found.digest.id}} {{ resource_found.name}} - {{ resource_found.detail}} + {{ resource_found.details}} {% for tag in resource_found.tags %} {{ tag.key}}: {{ tag.value}}
From 6f2c1af77026ed18c99ff3f3e76cf74896cdde4c Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sat, 27 Jun 2020 23:44:01 +0700 Subject: [PATCH 35/86] added listing of non-paginable resources --- cloudiscovery/provider/all/resource/all.py | 58 +++++++++++++++------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 5599ff1..01be161 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -1,8 +1,9 @@ import re from concurrent.futures.thread import ThreadPoolExecutor from functools import reduce -from typing import List +from typing import List, Optional +from botocore.exceptions import UnknownServiceError from botocore.loaders import Loader from shared.common import ( @@ -35,6 +36,12 @@ "aws_ssm_association", "aws_ssm_patch_baseline", "aws_ec2_prefix", + "aws_ec2_image", + "aws_ec2_region", + "aws_opsworks_operating_system", + "aws_rds_account_attribute", + "aws_route53_geo_location", + "aws_redshift_cluster_track", ] ON_TOP_POLICIES = [ @@ -132,6 +139,17 @@ def operation_allowed( return evaluation_result +def build_resource(base_resource, operation_name, resource_type) -> Optional[Resource]: + resource_name = retrieve_resource_name(base_resource, operation_name) + resource_id = retrieve_resource_id(base_resource, operation_name, resource_name) + + if resource_id is None or resource_name is None: + return None + return Resource( + digest=ResourceDigest(id=resource_id, type=resource_type), name=resource_name, + ) + + class AllResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): """ @@ -168,7 +186,12 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): resources = [] client = self.options.client(aws_service) service_model = boto_loader.load_service_model(aws_service, "service-2") - paginators_model = boto_loader.load_service_model(aws_service, "paginators-1") + try: + paginators_model = boto_loader.load_service_model( + aws_service, "paginators-1" + ) + except UnknownServiceError: + paginators_model = {"pagination": {}} service_full_name = service_model["metadata"]["serviceFullName"] message_handler( "Collecting data from {}...".format(service_full_name), "HEADER" @@ -244,28 +267,29 @@ def analyze_operation( for page in pages: if result_key is not None: page_resources = page[result_key] - else: + elif result_child in page[result_parent]: page_resources = page[result_parent][result_child] + else: + page_resources = [] for page_resource in page_resources: if isinstance(page_resource, str): continue - resource_name = retrieve_resource_name( - page_resource, operation_name + resource = build_resource( + page_resource, operation_name, resource_type ) - resource_id = retrieve_resource_id( - page_resource, operation_name, resource_name - ) - - if resource_id is None or resource_name is None: - continue - - resources.append( - Resource( - digest=ResourceDigest(id=resource_id, type=resource_type), - name=resource_name, + if resource is not None: + resources.append(resource) + else: + response = getattr(client, _to_snake_case(operation_name))() + for response_elem in response.values(): + if isinstance(response_elem, list): + for response_resource in response_elem: + resource = build_resource( + response_resource, operation_name, resource_type ) - ) + if resource is not None: + resources.append(resource) return resources def get_policies_allowed_actions(self): From a9d9759eeb45ecb892a7671539266e8aae5f3f01 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 28 Jun 2020 00:15:21 +0700 Subject: [PATCH 36/86] silenced default resources, more documentation --- README.md | 7 +++++-- cloudiscovery/provider/all/resource/all.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cdb4df3..ea50128 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,10 @@ Following resources are checked in IoT command: List all AWS resources (preview) -The command tries to call all AWS services (200+) and methods with name `Describe`, `Get...` and `List...`. +The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). + +The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). + ## Requirements and Installation @@ -137,7 +140,7 @@ pip install -U cloudiscovery aws configure ``` -### AWS Permissions +### AWS Permissions * The configured credentials must be associated to a user or role with proper permissions to do all checks. If you want to use a role with narrowed set of permissions just to perform cloud discovery, use a role from the following CF template shown below. To further increase security, you can add a block to check `aws:MultiFactorAuthPresent` condition in `AssumeRolePolicyDocument`. More on using IAM roles in the [configuration file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html). diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 01be161..8ffa067 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -27,6 +27,7 @@ "aws_polly_voice", "aws_lightsail_blueprint", "aws_lightsail_bundle", + "aws_lightsail_region", "aws_elastictranscoder_preset", "aws_ec2_vpc_endpoint_service", "aws_dms_endpoint_type", @@ -42,6 +43,11 @@ "aws_rds_account_attribute", "aws_route53_geo_location", "aws_redshift_cluster_track", + "aws_directconnect_location", + "aws_dms_account_attribute", + "aws_securityhub_standard", + "aws_ram_resource_type", + "aws_ram_permission", ] ON_TOP_POLICIES = [ From 433d0a4fa19916a5a708df4b6faae6c0d764a0db Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 28 Jun 2020 07:53:52 +0700 Subject: [PATCH 37/86] silenced multiple resources, fixing AWS engineers mistakes --- cloudiscovery/provider/all/resource/all.py | 84 +++++++++++++++++++--- 1 file changed, 73 insertions(+), 11 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 8ffa067..1ab0e61 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -1,6 +1,6 @@ +import functools import re from concurrent.futures.thread import ThreadPoolExecutor -from functools import reduce from typing import List, Optional from botocore.exceptions import UnknownServiceError @@ -13,10 +13,14 @@ ResourceDigest, message_handler, ResourceAvailable, + log_critical, ) -from shared.error_handler import exception OMITTED_RESOURCES = [ + "aws_cloudhsm_available_zone", + "aws_cloudhsm_hapg", + "aws_cloudhsm_hsm", + "aws_cloudhsm_luna_client", "aws_dax_default_parameter", "aws_dax_parameter_group", "aws_ec2_reserved_instances_offering", @@ -48,8 +52,38 @@ "aws_securityhub_standard", "aws_ram_resource_type", "aws_ram_permission", + "aws_ec2_account_attribute", + "aws_elasticbeanstalk_available_solution_stack", + "aws_redshift_account_attribute", + "aws_opsworks_user_profile", + "aws_directconnect_direct_connect_gateway_association", # DirectConnect resources endpoint are complicated + "aws_directconnect_direct_connect_gateway_attachment", + "aws_directconnect_interconnect", + "aws_dms_replication_task_assessment_result", + "aws_ec2_fpga_image", + "aws_ec2_launch_template_version", + "aws_ec2_reserved_instancesing", + "aws_ec2_spot_datafeed_subscription", + "aws_ec2_transit_gateway_multicast_domain", + "aws_elasticbeanstalk_configuration_option", ] +# Trying to fix documentation errors or its lack made by "happy pirates" at AWS +REQUIRED_PARAMS_OVERRIDE = { + "batch": {"ListJobs": ["jobQueue"]}, + "cloudformation": { + "DescribeStackEvents": ["stackName"], + "DescribeStackResources": ["stackName"], + "GetTemplate": ["stackName"], + "ListTypeVersions": ["arn"], + }, + "codecommit": {"GetBranch": ["repositoryName"],}, + "codedeploy": { + "GetDeploymentTarget": ["deploymentId"], + "ListDeploymentTargets": ["deploymentId"], + }, +} + ON_TOP_POLICIES = [ "kafka:ListClusters", "synthetics:DescribeCanaries", @@ -58,9 +92,11 @@ "ssm:GetParametersByPath", ] +PARALLEL_SERVICE_CALLS = 80 + -def _to_snake_case(function): - return reduce(lambda x, y: x + ("_" if y.isupper() else "") + y, function).lower() +def _to_snake_case(camel_case): + return re.sub("(?!^)([A-Z]+)", r"_\1", camel_case).lower() def last_singular_name_element(operation_name): @@ -146,6 +182,8 @@ def operation_allowed( def build_resource(base_resource, operation_name, resource_type) -> Optional[Resource]: + if isinstance(base_resource, str): + return None resource_name = retrieve_resource_name(base_resource, operation_name) resource_id = retrieve_resource_id(base_resource, operation_name, resource_name) @@ -156,6 +194,26 @@ def build_resource(base_resource, operation_name, resource_type) -> Optional[Res ) +def all_exception(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + # pylint: disable=broad-except + except Exception as e: + if func.__qualname__ == "AllResources.analyze_operation": + message = "\nError running operation {}, type {}. Error message {}".format( + args[2], args[1], str(e) + ) + else: + message = "\nError running method {}. Error message {}".format( + func.__qualname__, str(e) + ) + log_critical(message) + + return wrapper + + class AllResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): """ @@ -167,14 +225,14 @@ def __init__(self, options: BaseAwsOptions): self.options = options self.availabilityCheck = ResourceAvailable("") - @exception + @all_exception def get_resources(self) -> List[Resource]: boto_loader = Loader() aws_services = boto_loader.list_available_services(type_name="service-2") resources = [] allowed_actions = self.get_policies_allowed_actions() - with ThreadPoolExecutor(80) as executor: + with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor: results = executor.map( lambda aws_service: self.analyze_service( aws_service, boto_loader, allowed_actions @@ -187,7 +245,7 @@ def get_resources(self) -> List[Resource]: return resources - @exception + @all_exception def analyze_service(self, aws_service, boto_loader, allowed_actions): resources = [] client = self.options.client(aws_service) @@ -199,6 +257,8 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): except UnknownServiceError: paginators_model = {"pagination": {}} service_full_name = service_model["metadata"]["serviceFullName"] + # if service_full_name != 'AWS CloudFormation': + # return [] message_handler( "Collecting data from {}...".format(service_full_name), "HEADER" ) @@ -223,6 +283,11 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): input_model = service_model["shapes"][operation["input"]["shape"]] if "required" in input_model and input_model["required"]: continue + if ( + aws_service in REQUIRED_PARAMS_OVERRIDE + and operation["name"] in REQUIRED_PARAMS_OVERRIDE[aws_service] + ): + continue resource_type = "aws_{}_{}".format( aws_service, _to_snake_case( @@ -244,7 +309,7 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): resources.extend(analyze_operation) return resources - @exception + @all_exception # pylint: disable=too-many-locals def analyze_operation( self, resource_type, operation_name, has_paginator, client @@ -278,9 +343,6 @@ def analyze_operation( else: page_resources = [] for page_resource in page_resources: - if isinstance(page_resource, str): - continue - resource = build_resource( page_resource, operation_name, resource_type ) From 6da8205831deb9f055db289ed8c6e3cd68c7fb00 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 28 Jun 2020 09:26:38 +0700 Subject: [PATCH 38/86] aws all - adjusting eb, iam, iot, ml and rds methods --- cloudiscovery/provider/all/resource/all.py | 51 ++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 1ab0e61..7793c06 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -66,6 +66,16 @@ "aws_ec2_spot_datafeed_subscription", "aws_ec2_transit_gateway_multicast_domain", "aws_elasticbeanstalk_configuration_option", + "aws_elasticbeanstalk_platform_version", + "aws_iam_credential_report", + "aws_importexport_job", + "aws_iot_o_taupdate", + "aws_iot_default_authorizer", + "aws_workspaces_account", + "aws_workspaces_account_modification", + "aws_rds_export_task", + "aws_rds_custom_availability_zone", + "aws_rds_installation_media", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS @@ -77,11 +87,25 @@ "GetTemplate": ["stackName"], "ListTypeVersions": ["arn"], }, - "codecommit": {"GetBranch": ["repositoryName"],}, + "codecommit": {"GetBranch": ["repositoryName"]}, "codedeploy": { "GetDeploymentTarget": ["deploymentId"], "ListDeploymentTargets": ["deploymentId"], }, + "elasticbeanstalk": { + "DescribeEnvironmentHealth": ["environmentName"], + "DescribeEnvironmentManagedActionHistory": ["environmentName"], + "DescribeEnvironmentManagedActions": ["environmentName"], + "DescribeEnvironmentResources": ["environmentName"], + "DescribeInstancesHealth": ["environmentName"], + }, + "iam": { + "GetUser": ["userName"], + "ListAccessKeys": ["userName"], + "ListServiceSpecificCredentials": ["userName"], + "ListSigningCertificates": ["userName"], + }, + "iot": {"ListAuditFindings": ["taskId"]}, } ON_TOP_POLICIES = [ @@ -96,7 +120,28 @@ def _to_snake_case(camel_case): - return re.sub("(?!^)([A-Z]+)", r"_\1", camel_case).lower() + return ( + re.sub("(?!^)([A-Z]+)", r"_\1", camel_case) + .lower() + .replace("open_idconnect", "open_id_connect") + .replace("samlproviders", "saml_providers") + .replace("sshpublic_keys", "ssh_public_keys") + .replace("mfadevices", "mfa_devices") + .replace("cacertificates", "ca_certificates") + .replace("awsservice", "aws_service") + .replace("dbinstances", "db_instances") + .replace("drtaccess", "drt_access") + .replace("ipsets", "ip_sets") + .replace("mljobs", "ml_jobs") + .replace("dbcluster", "db_cluster") + .replace("dbengine", "db_engine") + .replace("dbsecurity", "db_security") + .replace("dbsubnet", "db_subnet") + .replace("dbsnapshot", "db_snapshot") + .replace("dbproxies", "db_proxies") + .replace("dbparameter", "db_parameter") + .replace("dbinstance", "db_instance") + ) def last_singular_name_element(operation_name): @@ -257,8 +302,6 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): except UnknownServiceError: paginators_model = {"pagination": {}} service_full_name = service_model["metadata"]["serviceFullName"] - # if service_full_name != 'AWS CloudFormation': - # return [] message_handler( "Collecting data from {}...".format(service_full_name), "HEADER" ) From 12e7061965c11540a1deec21c4318674c0964018 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 28 Jun 2020 22:54:17 +0700 Subject: [PATCH 39/86] aws all - improved error handling --- cloudiscovery/provider/all/resource/all.py | 82 +++++++++++++++++++--- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 7793c06..83be1d6 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -76,6 +76,11 @@ "aws_rds_export_task", "aws_rds_custom_availability_zone", "aws_rds_installation_media", + "aws_rds_d_bsecurity_group", + "aws_translate_text_translation_job", + "aws_rekognition_project", + "aws_rekognition_stream_processor", + "aws_sdb_domain", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS @@ -106,6 +111,31 @@ "ListSigningCertificates": ["userName"], }, "iot": {"ListAuditFindings": ["taskId"]}, + "opsworks": { + "ListAuditFindings": ["taskId"], + "DescribeAgentVersions": ["stackId"], + "DescribeApps": ["stackId"], + "DescribeCommands": ["deploymentId"], + "DescribeDeployments": ["appId"], + "DescribeEcsClusters": ["ecsClusterArns"], + "DescribeElasticIps": ["stackId"], + "DescribeElasticLoadBalancers": ["stackId"], + "DescribeInstances": ["stackId"], + "DescribeLayers": ["stackId"], + "DescribePermissions": ["stackId"], + "DescribeRaidArrays": ["stackId"], + "DescribeVolumes": ["stackId"], + }, + "ssm": {"DescribeMaintenanceWindowSchedule": ["windowId"],}, + "waf": { + "ListActivatedRulesInRuleGroup": ["ruleGroupId"], + "ListLoggingConfigurations": ["limit"], + }, + "waf-regional": { + "ListActivatedRulesInRuleGroup": ["ruleGroupId"], + "ListLoggingConfigurations": ["limit"], + }, + "wafv2": {"ListLoggingConfigurations": ["limit"],}, } ON_TOP_POLICIES = [ @@ -116,7 +146,7 @@ "ssm:GetParametersByPath", ] -PARALLEL_SERVICE_CALLS = 80 +PARALLEL_SERVICE_CALLS = 1 def _to_snake_case(camel_case): @@ -247,14 +277,40 @@ def wrapper(*args, **kwargs): # pylint: disable=broad-except except Exception as e: if func.__qualname__ == "AllResources.analyze_operation": - message = "\nError running operation {}, type {}. Error message {}".format( - args[2], args[1], str(e) - ) + exception_str = str(e) + if ( + "is not subscribed to AWS Security Hub" in exception_str + or "not enabled for securityhub" in exception_str + ): + message_handler( + "Operation {} not accessible, AWS Security Hub is not configured... Skipping".format( + args[2] + ), + "WARNING", + ) + elif ( + "not connect to the endpoint URL" in exception_str + or "not available in this region" in exception_str + or "API is not available" in exception_str + ): + message_handler( + "Service {} not available in the selected region... Skipping".format( + args[5] + ), + "WARNING", + ) + else: + log_critical( + "\nError running operation {}, type {}. Error message {}".format( + args[2], args[1], exception_str + ) + ) else: - message = "\nError running method {}. Error message {}".format( - func.__qualname__, str(e) + log_critical( + "\nError running method {}. Error message {}".format( + func.__qualname__, str(e) + ) ) - log_critical(message) return wrapper @@ -277,6 +333,12 @@ def get_resources(self) -> List[Resource]: resources = [] allowed_actions = self.get_policies_allowed_actions() + message_handler( + "Analyzing listing operations across {} service...".format( + len(aws_services) + ), + "HEADER", + ) with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor: results = executor.map( lambda aws_service: self.analyze_service( @@ -346,16 +408,16 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): if not operation_allowed(allowed_actions, aws_service, name): continue analyze_operation = self.analyze_operation( - resource_type, name, has_paginator, client + resource_type, name, has_paginator, client, service_full_name ) if analyze_operation is not None: resources.extend(analyze_operation) return resources @all_exception - # pylint: disable=too-many-locals + # pylint: disable=too-many-locals,too-many-arguments def analyze_operation( - self, resource_type, operation_name, has_paginator, client + self, resource_type, operation_name, has_paginator, client, service_full_name ) -> List[Resource]: resources = [] if has_paginator: From 9cc90e8602aba9fa8486ef8fc8a7cee535db603d Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 28 Jun 2020 22:56:46 +0100 Subject: [PATCH 40/86] Stable version of aws limits check --- README.md | 15 ++- cloudiscovery/provider/limits/command.py | 8 +- cloudiscovery/provider/limits/resource/all.py | 52 +++++--- cloudiscovery/shared/common.py | 12 +- cloudiscovery/shared/common_aws.py | 122 +++++++++++++++--- cloudiscovery/shared/report.py | 63 ++++++--- cloudiscovery/templates/report_limits.html | 46 +++++++ 7 files changed, 257 insertions(+), 61 deletions(-) create mode 100644 cloudiscovery/templates/report_limits.html diff --git a/README.md b/README.md index cdb4df3..4d04e0f 100644 --- a/README.md +++ b/README.md @@ -219,16 +219,22 @@ cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filt cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] ``` -1.3 To detect all AWS resources: +1.4 To detect all AWS resources: ```sh cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] ``` +1.5 To check AWS limits per resource: + +```sh +cloudiscovery aws-limits --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] +``` + 2. For help use: ```sh -cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all] -h +cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all|aws-limits] -h ``` ### Filtering @@ -247,6 +253,11 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a 2. `aws:cloudformation:stack-id` - Stack id 3. `aws:cloudformation:logical-id` - Logical id defined in CF template +### Limits usage + +AWS has a default quota to all services. At the first time that you create an account, AWS apply this default quota to all services. +You can ask to increase the quota value of a certain service via ticket and this script will detect this. + ### Using a Docker container To build docker container using Dockerfile diff --git a/cloudiscovery/provider/limits/command.py b/cloudiscovery/provider/limits/command.py index 81a8df4..1742d15 100644 --- a/cloudiscovery/provider/limits/command.py +++ b/cloudiscovery/provider/limits/command.py @@ -1,6 +1,7 @@ from shared.command import BaseCommand, CommandRunner from shared.common import BaseAwsOptions from shared.diagram import NoDiagram +from shared.common_aws import ALLOWED_SERVICES_CODES class Limits(BaseCommand): @@ -13,7 +14,12 @@ def __init__(self, region_names, session, services): :param services: """ super().__init__(region_names, session, False, False) - self.services = services + self.services = [] + if services is None: + for service in ALLOWED_SERVICES_CODES: + self.services.append(service) + else: + self.services = services.split(",") def run(self): diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index 526f3e7..dc5c2dd 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -8,6 +8,7 @@ ResourceDigest, message_handler, ResourceCache, + LimitsValues, ) from shared.error_handler import exception @@ -30,7 +31,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - services = self.options.services.split(",") + services = self.options.services + for service in services: cache_key = "aws_limits_" + service + "_" + self.options.region_name cache = self.cache.get_key(cache_key) @@ -40,39 +42,59 @@ def get_resources(self) -> List[Resource]: data_quota_code["quota_code"] ] + value_aws = data_quota_code["value"] + # Quota is adjustable by ticket request, then must override this values if bool(data_quota_code["adjustable"]) is True: - response_quota = client_quota.get_service_quota( - ServiceCode=service, QuotaCode=data_quota_code["quota_code"] - ) - if "Value" in response_quota["Quota"]: - value = response_quota["Quota"]["Value"] - else: + try: + response_quota = client_quota.get_service_quota( + ServiceCode=service, QuotaCode=data_quota_code["quota_code"] + ) + if "Value" in response_quota["Quota"]: + value = response_quota["Quota"]["Value"] + else: + value = data_quota_code["value"] + except client_quota.exceptions.NoSuchResourceException: value = data_quota_code["value"] message_handler( "Collecting data from Quota: " + + service + + " - " + data_quota_code["quota_name"] + "...", "HEADER", ) - client = self.options.session.client(service) + + """ + TODO: Add this as alias to convert service name + """ + if service == "elasticloadbalancing": + service = "elbv2" + + client = self.options.session.client( + service, region_name=self.options.region_name + ) response = getattr(client, quota_data["method"])() - total = str(len(response[quota_data["key"]])) + usage = len(response[quota_data["key"]]) resources_found.append( Resource( digest=ResourceDigest( id=data_quota_code["quota_code"], type="aws_limit" ), - name=data_quota_code["quota_name"], - group=service, - details="Limit: " - + str(int(value)) - + "... Current usage: " - + total, + name="", + group="", + limits=LimitsValues( + quota_name=data_quota_code["quota_name"], + quota_code=data_quota_code["quota_code"], + aws_limit=int(value_aws), + local_limit=int(value), + usage=int(usage), + service=service, + ), ) ) diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 1e4c63a..9e9e474 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -66,6 +66,15 @@ class Filterable: pass +class LimitsValues(NamedTuple): + service: str + quota_name: str + quota_code: str + aws_limit: int + local_limit: int + usage: int + + class ResourceTag(NamedTuple, Filterable): key: str value: str @@ -77,10 +86,11 @@ class ResourceType(NamedTuple, Filterable): class Resource(NamedTuple): digest: ResourceDigest - name: str + name: str = "" details: str = "" group: str = "" tags: List[ResourceTag] = [] + limits: List[LimitsValues] = [] class ResourceCache: diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 7b9d167..b644e5e 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -5,15 +5,75 @@ SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) ALLOWED_SERVICES_CODES = { + "acm": { + "L-F141DD1D": { + "method": "list_certificates", + "key": "CertificateSummaryList", + "fields": [], + }, + "global": False, + }, + "codebuild": { + "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, + "global": False, + }, + "codecommit": { + "L-81790602": { + "method": "list_repositories", + "key": "repositories", + "fields": [], + }, + "global": False, + }, + "cloudformation": { + "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, + "global": False, + }, "ec2": { "L-0263D0A3": { "method": "describe_addresses", "key": "Addresses", - "fields": ["PublicIp"], - } + "fields": [], + }, + "global": False, }, - "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []} + "elasticbeanstalk": { + "L-8EFC1C51": { + "method": "describe_environments", + "key": "Environments", + "fields": [], + }, + "L-1CEABD17": { + "method": "describe_applications", + "key": "Applications", + "fields": [], + }, + "global": False, + }, + "elasticloadbalancing": { + "L-53DA6B97": { + "method": "describe_load_balancers", + "key": "LoadBalancers", + "fields": [], + }, + "global": False, + }, + "route53": { + "L-4EA4796A": { + "method": "list_hosted_zones", + "key": "HostedZones", + "fields": [], + }, + "L-ACB674F3": { + "method": "list_health_checks", + "key": "HealthChecks", + "fields": [], + }, + "global": True, + }, + "s3": { + "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, + "global": False, }, } @@ -37,9 +97,14 @@ def describe_subnet(vpc_options, subnet_ids): class LimitParameters: def __init__(self, session, region: str, services): self.region = region - self.session = session.client("service-quotas", region_name=region) self.cache = ResourceCache() - self.services = services.split(",") + self.session = session + self.services = [] + if services is None: + for service in ALLOWED_SERVICES_CODES: + self.services.append(service) + else: + self.services = services def init_globalaws_limits_cache(self): """ @@ -64,21 +129,36 @@ def init_globalaws_limits_cache(self): cache_codes = dict() for quota_code in ALLOWED_SERVICES_CODES[service_code]: - response = self.session.get_aws_default_service_quota( - ServiceCode=service_code, QuotaCode=quota_code - ) - - item_to_add = { - "valye": response["Quota"]["Value"], - "adjustable": response["Quota"]["Adjustable"], - "quota_code": quota_code, - "quota_name": response["Quota"]["QuotaName"], - } - - if service_code in cache_codes: - cache_codes[service_code].append(item_to_add) - else: - cache_codes[service_code] = [item_to_add] + + if quota_code != "global": + """ + Impossible to instance once at __init__ method. + Global services such route53 MUST USE us-east-1 region + """ + if ALLOWED_SERVICES_CODES[service_code]["global"]: + service_quota = self.session.client( + "service-quotas", region_name="us-east-1" + ) + else: + service_quota = self.session.client( + "service-quotas", region_name=self.region + ) + + response = service_quota.get_aws_default_service_quota( + ServiceCode=service_code, QuotaCode=quota_code + ) + + item_to_add = { + "value": response["Quota"]["Value"], + "adjustable": response["Quota"]["Adjustable"], + "quota_code": quota_code, + "quota_name": response["Quota"]["QuotaName"], + } + + if service_code in cache_codes: + cache_codes[service_code].append(item_to_add) + else: + cache_codes[service_code] = [item_to_add] self.cache.set_key(key=cache_key, value=cache_codes, expire=1296000) diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 9edbcc8..709e182 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -26,25 +26,40 @@ def general_report( message_handler("\n\nFound resources", "HEADER") for resource in resources: - message = "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( - resource.digest.type, - resource.digest.id, - resource.name, - resource.details, - ) + # Report to limits + if resource.limits: + percent = (resource.limits.usage / resource.limits.local_limit) * 100 + usage = str(resource.limits.usage) + " - " + str(percent) + "%" + message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} \ + - applied quota: {} - usage: {}".format( + resource.limits.service, + resource.limits.quota_code, + resource.limits.quota_name, + resource.limits.aws_limit, + resource.limits.local_limit, + usage, + ) + else: + message = "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + resource.digest.type, + resource.digest.id, + resource.name, + resource.details, + ) message_handler(message, "OKBLUE") - message_handler("\n\nFound relations", "HEADER") - for resource_relation in resource_relations: - message = "resource type: {} - resource id: {} -> resource type: {} - resource id: {}".format( - resource_relation.from_node.type, - resource_relation.from_node.id, - resource_relation.to_node.type, - resource_relation.to_node.id, - ) + if resource_relations: + message_handler("\n\nFound relations", "HEADER") + for resource_relation in resource_relations: + message = "resource type: {} - resource id: {} -> resource type: {} - resource id: {}".format( + resource_relation.from_node.type, + resource_relation.from_node.id, + resource_relation.to_node.type, + resource_relation.to_node.id, + ) - message_handler(message, "OKBLUE") + message_handler(message, "OKBLUE") @exception def html_report( @@ -69,12 +84,18 @@ def html_report( with open(image_name, "rb") as image_file: diagram_image = base64.b64encode(image_file.read()).decode("utf-8") - html_output = dir_template.get_template("report_html.html").render( - default_name=title, - resources_found=resources, - resources_relations=resource_relations, - diagram_image=diagram_image, - ) + if resources: + if resources[0].limits: + html_output = dir_template.get_template("report_limits.html").render( + default_name=title, resources_found=resources + ) + else: + html_output = dir_template.get_template("report_html.html").render( + default_name=title, + resources_found=resources, + resources_relations=resource_relations, + diagram_image=diagram_image, + ) self.make_directories() diff --git a/cloudiscovery/templates/report_limits.html b/cloudiscovery/templates/report_limits.html new file mode 100644 index 0000000..a1ab338 --- /dev/null +++ b/cloudiscovery/templates/report_limits.html @@ -0,0 +1,46 @@ +

cloudiscovery - A tool to help you discover resources in the cloud environment.

+

{{ default_name }}

+

Limits

+ + + + + + + + + + + +{% for resource_found in resources_found %} + {% set percent = (resource_found.limits.usage / resource_found.limits.local_limit) * 100 %} + + {% if percent <= 70 %} + {% set color = "rgb(0,128,0)" %} + {% set message = "OK" %} + {% elif percent > 70 and percent <= 90 %} + {% set color = "rgb(0,0,139)" %} + {% set message = "Attention" %} + {% elif percent > 90 %} + {% set color = "rgb(255,0,0)" %} + {% set message = "Risk" %} + {% endif %} + + + + + + + + + +{%- endfor %} + +
ServiceQuota codeQuota nameAWS default quotaApplied quotaUsageUsage percent
{{ resource_found.limits.service}}{{ resource_found.limits.quota_code}}{{ resource_found.limits.quota_name}}{{ resource_found.limits.aws_limit}}{{ resource_found.limits.local_limit}}{{ resource_found.limits.usage}}{{ percent }}% - {{ message }} + + + Sorry, your browser does not support inline SVG. + +
+ +

 

\ No newline at end of file From e4aa3f04b740e1fb9a645772da3ed0e9dbd5d213 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 28 Jun 2020 23:00:11 +0100 Subject: [PATCH 41/86] Stable version of aws limits check --- cloudiscovery/shared/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 709e182..349c582 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -30,8 +30,8 @@ def general_report( if resource.limits: percent = (resource.limits.usage / resource.limits.local_limit) * 100 usage = str(resource.limits.usage) + " - " + str(percent) + "%" - message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} \ - - applied quota: {} - usage: {}".format( + # pylint: disable=line-too-long + message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 resource.limits.service, resource.limits.quota_code, resource.limits.quota_name, From 3cee6941dc324b5b4bcd8ed479134a0c3856d9ee Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 28 Jun 2020 23:12:35 +0100 Subject: [PATCH 42/86] Added amplify apps --- cloudiscovery/shared/common_aws.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index b644e5e..6c02121 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -13,6 +13,10 @@ }, "global": False, }, + "amplify": { + "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, + "global": False, + }, "codebuild": { "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, "global": False, From e3140ccac43f66986bc13bdf7a559382c84f814d Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 10:39:38 +0100 Subject: [PATCH 43/86] Code refactor and added new limits checks --- README.md | 2 + cloudiscovery/provider/limits/resource/all.py | 97 ++++++++++++++++++- cloudiscovery/shared/common.py | 1 + cloudiscovery/shared/common_aws.py | 77 +-------------- cloudiscovery/shared/report.py | 8 +- cloudiscovery/templates/report_limits.html | 2 +- 6 files changed, 107 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index a54fa02..271f626 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,8 @@ To run pre-commit hooks, you can issue the following command: pre-commit run --all-files ``` +To add new resources to check limit, please remove "assets/.cache/cache.db" + ## Making a release 1. Update the version in cloudiscovery/__init\__.py and create a new git tag with `git tag $VERSION`. diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index dc5c2dd..3cd0cd8 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -1,6 +1,5 @@ from typing import List -from shared.common_aws import ALLOWED_SERVICES_CODES from shared.common import ( ResourceProvider, Resource, @@ -12,6 +11,98 @@ ) from shared.error_handler import exception +ALLOWED_SERVICES_CODES = { + "acm": { + "L-F141DD1D": { + "method": "list_certificates", + "key": "CertificateSummaryList", + "fields": [], + }, + "global": False, + }, + "amplify": { + "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, + "global": False, + }, + "codebuild": { + "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, + "global": False, + }, + "codecommit": { + "L-81790602": { + "method": "list_repositories", + "key": "repositories", + "fields": [], + }, + "global": False, + }, + "cloudformation": { + "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, + "global": False, + }, + "ec2": { + "L-0263D0A3": { + "method": "describe_addresses", + "key": "Addresses", + "fields": [], + }, + "global": False, + }, + "elasticbeanstalk": { + "L-8EFC1C51": { + "method": "describe_environments", + "key": "Environments", + "fields": [], + }, + "L-1CEABD17": { + "method": "describe_applications", + "key": "Applications", + "fields": [], + }, + "global": False, + }, + "elasticloadbalancing": { + "L-53DA6B97": { + "method": "describe_load_balancers", + "key": "LoadBalancers", + "fields": [], + }, + "global": False, + }, + "iam": { + "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, + "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, + "L-BF35879D": { + "method": "list_server_certificates", + "key": "ServerCertificateMetadataList", + "fields": [], + }, + "L-6E65F664": { + "method": "list_instance_profiles", + "key": "InstanceProfiles", + "fields": [], + }, + "global": True, + }, + "route53": { + "L-4EA4796A": { + "method": "list_hosted_zones", + "key": "HostedZones", + "fields": [], + }, + "L-ACB674F3": { + "method": "list_health_checks", + "key": "HealthChecks", + "fields": [], + }, + "global": True, + }, + "s3": { + "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, + "global": False, + }, +} + class LimitResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): @@ -25,6 +116,7 @@ def __init__(self, options: BaseAwsOptions): self.cache = ResourceCache() @exception + # pylint: disable=too-many-locals def get_resources(self) -> List[Resource]: client_quota = self.options.session.client("service-quotas") @@ -80,6 +172,8 @@ def get_resources(self) -> List[Resource]: usage = len(response[quota_data["key"]]) + percent = round((usage / value) * 100, 2) + resources_found.append( Resource( digest=ResourceDigest( @@ -94,6 +188,7 @@ def get_resources(self) -> List[Resource]: local_limit=int(value), usage=int(usage), service=service, + percent=percent, ), ) ) diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 9e9e474..2c578c6 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -73,6 +73,7 @@ class LimitsValues(NamedTuple): aws_limit: int local_limit: int usage: int + percent: float class ResourceTag(NamedTuple, Filterable): diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 6c02121..441d35e 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -2,84 +2,9 @@ from cachetools import TTLCache from shared.common import ResourceCache, message_handler +from provider.limits.resource.all import ALLOWED_SERVICES_CODES SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) -ALLOWED_SERVICES_CODES = { - "acm": { - "L-F141DD1D": { - "method": "list_certificates", - "key": "CertificateSummaryList", - "fields": [], - }, - "global": False, - }, - "amplify": { - "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, - "global": False, - }, - "codebuild": { - "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, - "global": False, - }, - "codecommit": { - "L-81790602": { - "method": "list_repositories", - "key": "repositories", - "fields": [], - }, - "global": False, - }, - "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, - "global": False, - }, - "ec2": { - "L-0263D0A3": { - "method": "describe_addresses", - "key": "Addresses", - "fields": [], - }, - "global": False, - }, - "elasticbeanstalk": { - "L-8EFC1C51": { - "method": "describe_environments", - "key": "Environments", - "fields": [], - }, - "L-1CEABD17": { - "method": "describe_applications", - "key": "Applications", - "fields": [], - }, - "global": False, - }, - "elasticloadbalancing": { - "L-53DA6B97": { - "method": "describe_load_balancers", - "key": "LoadBalancers", - "fields": [], - }, - "global": False, - }, - "route53": { - "L-4EA4796A": { - "method": "list_hosted_zones", - "key": "HostedZones", - "fields": [], - }, - "L-ACB674F3": { - "method": "list_health_checks", - "key": "HealthChecks", - "fields": [], - }, - "global": True, - }, - "s3": { - "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, - "global": False, - }, -} def describe_subnet(vpc_options, subnet_ids): diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 349c582..82154fe 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -28,8 +28,12 @@ def general_report( for resource in resources: # Report to limits if resource.limits: - percent = (resource.limits.usage / resource.limits.local_limit) * 100 - usage = str(resource.limits.usage) + " - " + str(percent) + "%" + usage = ( + str(resource.limits.usage) + + " - " + + str(resource.limits.percent) + + "%" + ) # pylint: disable=line-too-long message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 resource.limits.service, diff --git a/cloudiscovery/templates/report_limits.html b/cloudiscovery/templates/report_limits.html index a1ab338..22fcdda 100644 --- a/cloudiscovery/templates/report_limits.html +++ b/cloudiscovery/templates/report_limits.html @@ -13,7 +13,7 @@ Usage percent {% for resource_found in resources_found %} - {% set percent = (resource_found.limits.usage / resource_found.limits.local_limit) * 100 %} + {% set percent = resource_found.limits.percent %} {% if percent <= 70 %} {% set color = "rgb(0,128,0)" %} From ad91e1b87d329686f85028d0cc51cec48652d7d9 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 11:00:48 +0100 Subject: [PATCH 44/86] New limits checks --- cloudiscovery/provider/limits/resource/all.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index 3cd0cd8..0e566d0 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -48,6 +48,10 @@ }, "global": False, }, + "ecs": { + "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, + "global": False, + }, "elasticbeanstalk": { "L-8EFC1C51": { "method": "describe_environments", @@ -82,8 +86,13 @@ "key": "InstanceProfiles", "fields": [], }, + "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, "global": True, }, + "kms": { + "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "global": False, + }, "route53": { "L-4EA4796A": { "method": "list_hosted_zones", @@ -101,6 +110,10 @@ "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, "global": False, }, + "sns": { + "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, + "global": False, + }, } From 0cdd054b66e0ebc468d88307562ed4171f964139 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 13:09:28 +0100 Subject: [PATCH 45/86] New limits checks --- cloudiscovery/provider/limits/resource/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index 0e566d0..b88208e 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -172,7 +172,7 @@ def get_resources(self) -> List[Resource]: ) """ - TODO: Add this as alias to convert service name + TODO: Add this as alias to convert service name. """ if service == "elasticloadbalancing": service = "elbv2" From abac1aaea7c65dea076fb12199f08c9e585c95e0 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 13:48:19 +0100 Subject: [PATCH 46/86] New limits checks --- cloudiscovery/provider/limits/resource/all.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index b88208e..dc26c7f 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -106,6 +106,24 @@ }, "global": True, }, + "rds": { + "L-7B6409FD": { + "method": "describe_db_instances", + "key": "DBInstances", + "fields": [], + }, + "L-952B80B8": { + "method": "describe_db_clusters", + "key": "DBClusters", + "fields": [], + }, + "L-DE55804A": { + "method": "describe_db_parameter_groups", + "key": "DBParameterGroups", + "fields": [], + }, + "global": False, + }, "s3": { "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, "global": False, From 72edf7354225af9b465c3a47c8cf78f2599ec865 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 15:54:01 +0100 Subject: [PATCH 47/86] New limits checks and report name fix --- cloudiscovery/provider/limits/command.py | 6 +++--- cloudiscovery/provider/limits/resource/all.py | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cloudiscovery/provider/limits/command.py b/cloudiscovery/provider/limits/command.py index 1742d15..81458c2 100644 --- a/cloudiscovery/provider/limits/command.py +++ b/cloudiscovery/provider/limits/command.py @@ -25,16 +25,16 @@ def run(self): for region in self.region_names: self.init_globalaws_limits_cache(region=region, services=self.services) - options = BaseAwsOptions( + limit_options = BaseAwsOptions( session=self.session, region_name=region, services=self.services ) command_runner = CommandRunner(services=self.services) command_runner.run( provider="limits", - options=options, + options=limit_options, diagram_builder=NoDiagram(), title="AWS Limits - Region {}".format(region), # pylint: disable=no-member - filename="AWS Limits - Region {}".format(region), + filename=limit_options.resulting_file_name("limits"), ) diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index dc26c7f..a7c5238 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -40,6 +40,10 @@ "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, "global": False, }, + "dynamodb": { + "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": []}, + "global": False, + }, "ec2": { "L-0263D0A3": { "method": "describe_addresses", @@ -122,6 +126,11 @@ "key": "DBParameterGroups", "fields": [], }, + "L-9FA33840": { + "method": "describe_option_groups", + "key": "OptionGroupsList", + "fields": [], + }, "global": False, }, "s3": { From b85a6a0edf58de1ab755fc541f91e0cf5006fb85 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 16:54:38 +0100 Subject: [PATCH 48/86] Updated doc and new checks --- README.md | 26 +++++++++++++++++-- cloudiscovery/provider/limits/resource/all.py | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 271f626..0e17d1f 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,8 @@ cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filt cloudiscovery aws-limits --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] ``` +Check [limits usage](#limits-usage) section. + 2. For help use: ```sh @@ -258,8 +260,28 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a ### Limits usage -AWS has a default quota to all services. At the first time that you create an account, AWS apply this default quota to all services. -You can ask to increase the quota value of a certain service via ticket and this script will detect this. +It's possible to check resources limits in an account. This script allows check all services availables or check only a specific resource. Using `--services value,value,value` filter, you can inform all services that want to check. + +- Services available + - acm + - amplify + - codebuild + - codecommit + - cloudformation + - dynamodb + - ec2 + - ecs + - elasticbeanstalk + - elasticloadbalancing + - iam + - route53 + - rds + - s3 + - sns + + +AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. +An administrator can ask to increase the quota value of a certain service via ticket and this script will detect this. ### Using a Docker container diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limits/resource/all.py index a7c5238..6a3f956 100644 --- a/cloudiscovery/provider/limits/resource/all.py +++ b/cloudiscovery/provider/limits/resource/all.py @@ -38,6 +38,7 @@ }, "cloudformation": { "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, + "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": []}, "global": False, }, "dynamodb": { From 1a0883c4e0ef0ee6145c7f6804d12e3bf942a665 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 16:58:44 +0100 Subject: [PATCH 49/86] Readme issues --- README.md | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 0e17d1f..2562d2c 100644 --- a/README.md +++ b/README.md @@ -263,22 +263,21 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a It's possible to check resources limits in an account. This script allows check all services availables or check only a specific resource. Using `--services value,value,value` filter, you can inform all services that want to check. - Services available - - acm - - amplify - - codebuild - - codecommit - - cloudformation - - dynamodb - - ec2 - - ecs - - elasticbeanstalk - - elasticloadbalancing - - iam - - route53 - - rds - - s3 - - sns - + * acm + * amplify + * codebuild + * codecommit + * cloudformation + * dynamodb + * ec2 + * ecs + * elasticbeanstalk + * elasticloadbalancing + * iam + * route53 + * rds + * s3 + * sns AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. An administrator can ask to increase the quota value of a certain service via ticket and this script will detect this. From ec52a699e066078f89cf9b1560f12671206e4bee Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 17:01:22 +0100 Subject: [PATCH 50/86] Readme issues --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2562d2c..a4a6196 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a It's possible to check resources limits in an account. This script allows check all services availables or check only a specific resource. Using `--services value,value,value` filter, you can inform all services that want to check. -- Services available +* Services available * acm * amplify * codebuild From e5426e8fe3840695cc528f4b59a87c51e1d1c658 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 29 Jun 2020 23:39:39 +0700 Subject: [PATCH 51/86] #93 review --- README.md | 6 ++++-- cloudiscovery/shared/common.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a4a6196..af0d105 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,9 @@ aws configure "synthetics:DescribeCanaries", "medialive:ListInputs", "cloudhsm:DescribeClusters", - "ssm:GetParametersByPath" + "ssm:GetParametersByPath", + "servicequotas:Get*", + "amplify:ListApps" ], "Resource": [ "*" ] } @@ -260,7 +262,7 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a ### Limits usage -It's possible to check resources limits in an account. This script allows check all services availables or check only a specific resource. Using `--services value,value,value` filter, you can inform all services that want to check. +It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. With `--services value,value,value` selection, you can narrow down checks to services that you want to check. * Services available * acm diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 2c578c6..70dc2f2 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -101,7 +101,7 @@ def __init__(self): + "/../../assets/.cache/" ) - def set_key(self, key: str, value: int, expire: int): + def set_key(self, key: str, value: object, expire: int): self.cache.set(key=key, value=value, expire=expire) def get_key(self, key: str): From b5ab91046a57b4155d60813ca6ecb497450d2d45 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 00:46:57 +0700 Subject: [PATCH 52/86] command name adjustment, readme adjustment, all command type naming fix, addressed all command errors --- README.md | 254 +++++++++--------- cloudiscovery/__init__.py | 8 +- cloudiscovery/provider/all/resource/all.py | 75 ++++-- .../provider/{limits => limit}/__init__.py | 0 .../provider/{limits => limit}/command.py | 26 +- .../{limits => limit}/resource/__init__.py | 0 .../{limits => limit}/resource/all.py | 0 cloudiscovery/shared/common.py | 1 - cloudiscovery/shared/common_aws.py | 10 +- cloudiscovery/shared/report.py | 2 +- 10 files changed, 221 insertions(+), 155 deletions(-) rename cloudiscovery/provider/{limits => limit}/__init__.py (100%) rename cloudiscovery/provider/{limits => limit}/command.py (71%) rename cloudiscovery/provider/{limits => limit}/resource/__init__.py (100%) rename cloudiscovery/provider/{limits => limit}/resource/all.py (100%) diff --git a/README.md b/README.md index af0d105..41a6aef 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ ![aws provider](https://img.shields.io/badge/provider-AWS-orange?logo=amazon-aws&color=ff9900) -Cloudiscovery helps you to analyze resources in your cloud (AWS/GCP/Azure/Alibaba/IBM) account. Now this tool only can check resources in AWS, but we are working to expand to other providers. +Cloudiscovery helps you to analyze resources in your cloud (AWS/GCP/Azure/Alibaba/IBM) account. Now this tool only can check resources in AWS, but we are working to expand to other providers. + +The tool consists of various commands to help you understand the cloud infrastructure. ## Features @@ -22,105 +24,49 @@ Commands can generate diagrams. When modelling them, we try to follow the follow Edward Tufte -### AWS VPC - -Example of a diagram: +## Report -![diagrams logo](docs/assets/aws-vpc.png) +The commands generate reports that can be used to analyze command reports. -Following resources are checked in VPC command: +### CLI -* Autoscaling Group -* Classic/Network/Application Load Balancer -* Client VPN Endpoints -* CloudHSM -* DocumentDB -* Directory Service -* EC2 Instance -* ECS -* EFS -* ElastiCache -* Elasticsearch -* EKS -* EMR -* IAM Policy -* Internet Gateway (IGW) -* Lambda -* Media Connect -* Media Live -* Media Store Policy -* MSK -* NACL -* NAT Gateway -* Neptune -* QuickSight -* RDS -* REST Api Policy -* Route Table -* S3 Policy -* Sagemaker Notebook -* Sagemaker Training Job -* Sagemaker Model -* Security Group -* SQS Queue Policy -* Site-to-Site VPN Connections -* Subnet -* Synthetic Canary -* VPC Peering -* VPC Endpoint -* VPN Customer Gateways -* Virtual Private Gateways -* Workspace - -The subnets are aggregated to simplify the diagram and hide infrastructure redundancies. There can be two types of subnet aggregates: -1. Private* ones with a route `0.0.0.0/0` to Internet Gateway -2. Public* ones without any route to IGW - -If EC2 instances and ECS instances are part of an autoscaling group, those instances will be aggregated on a diagram. - -### AWS Policy - -Example of a diagram: - -![diagrams logo](docs/assets/aws-policy.png) - -Following resources are checked in Policy command: - -* [AWS Principal](https://gist.github.com/shortjared/4c1e3fe52bdfa47522cfe5b41e5d6f22) that are able to assume roles -* IAM Group -* IAM Group to policy relationship -* IAM Policy -* IAM Role -* IAM Role to policy relationship -* IAM User -* IAM User to group relationship -* IAM User to policy relationship +1. Run the cloudiscovery command with following options (if a region not informed, this script will try to get from ~/.aws/credentials): -Some roles can be aggregated to simplify the diagram. If a role is associated with a principal and is not attached to any named policy, will be aggregated. +1.1 To detect AWS VPC resources: -### AWS IoT +```sh +cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] +``` +1.2 To detect AWS policy resources: -Example of a diagram: +```sh +cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] +``` +1.3 To detect AWS IoT resources: -![diagrams logo](docs/assets/aws-iot.png) +```sh +cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] +``` -Following resources are checked in IoT command: +1.4 To detect all AWS resources: -* IoT Billing Group -* IoT Certificates -* IoT Jobs -* IoT Policies -* IoT Thing -* IoT Thing Type +```sh +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] +``` -### AWS All +1.5 To check AWS limits per resource: -List all AWS resources (preview) +```sh +cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] +``` -The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). +Check [limits usage](#limits-usage) section. -The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). +2. For help use: +```sh +cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all|aws-limit] -h +``` ## Requirements and Installation @@ -204,61 +150,123 @@ aws configure * (Optional) If you want to be able to switch between multiple AWS credentials and settings, you can configure [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) and later pass profile name when running the tool. -### Usage +### Filtering -1. Run the cloudiscovery command with following options (if a region not informed, this script will try to get from ~/.aws/credentials): +It's possible to filter resources by tags and resource type. To filter, add an option `--filter `, where `` can be: -1.1 To detect AWS VPC resources: +1. `Name=tags.costCenter;Value=20000` - to filter resources by a tag name `costCenter` and with value `20000`. +2. `Name=type;Value=aws_lambda_function` to only list lambda functions. -```sh -cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] -``` -1.2 To detect AWS policy resources: +It's possible to pass multiple values, to be able to select a value from a set. Values are split by `:` sign. If a desired value has a `:` sign, wrap it in `'` signs e.g. `--filter="Name=tags.costCenter;Value=20000:'20001:1'`. -```sh -cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] -``` -1.3 To detect AWS IoT resources: +It is possible to pass multiple filter options, just pass `-f filter_1 -f filter_2`. In that case, the tool will return resources that match either of the filters -```sh -cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] -``` +Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-aws-cloudformation-stack/): +1. `aws:cloudformation:stack-name` - Stack name +2. `aws:cloudformation:stack-id` - Stack id +3. `aws:cloudformation:logical-id` - Logical id defined in CF template -1.4 To detect all AWS resources: -```sh -cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] -``` +### AWS VPC -1.5 To check AWS limits per resource: +Example of a diagram: -```sh -cloudiscovery aws-limits --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] -``` +![diagrams logo](docs/assets/aws-vpc.png) -Check [limits usage](#limits-usage) section. +Following resources are checked in VPC command: -2. For help use: +* Autoscaling Group +* Classic/Network/Application Load Balancer +* Client VPN Endpoints +* CloudHSM +* DocumentDB +* Directory Service +* EC2 Instance +* ECS +* EFS +* ElastiCache +* Elasticsearch +* EKS +* EMR +* IAM Policy +* Internet Gateway (IGW) +* Lambda +* Media Connect +* Media Live +* Media Store Policy +* MSK +* NACL +* NAT Gateway +* Neptune +* QuickSight +* RDS +* REST Api Policy +* Route Table +* S3 Policy +* Sagemaker Notebook +* Sagemaker Training Job +* Sagemaker Model +* Security Group +* SQS Queue Policy +* Site-to-Site VPN Connections +* Subnet +* Synthetic Canary +* VPC Peering +* VPC Endpoint +* VPN Customer Gateways +* Virtual Private Gateways +* Workspace -```sh -cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all|aws-limits] -h -``` +The subnets are aggregated to simplify the diagram and hide infrastructure redundancies. There can be two types of subnet aggregates: +1. Private* ones with a route `0.0.0.0/0` to Internet Gateway +2. Public* ones without any route to IGW -### Filtering +If EC2 instances and ECS instances are part of an autoscaling group, those instances will be aggregated on a diagram. -It's possible to filter resources by tags and resource type. To filter, add an option `--filter `, where `` can be: +### AWS Policy -1. `Name=tags.costCenter;Value=20000` - to filter resources by a tag name `costCenter` and with value `20000`. -2. `Name=type;Value=aws_lambda_function` to only list lambda functions. +Example of a diagram: -It's possible to pass multiple values, to be able to select a value from a set. Values are split by `:` sign. If a desired value has a `:` sign, wrap it in `'` signs e.g. `--filter="Name=tags.costCenter;Value=20000:'20001:1'`. +![diagrams logo](docs/assets/aws-policy.png) -It is possible to pass multiple filter options, just pass `-f filter_1 -f filter_2`. In that case, the tool will return resources that match either of the filters +Following resources are checked in Policy command: -Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-aws-cloudformation-stack/): -1. `aws:cloudformation:stack-name` - Stack name -2. `aws:cloudformation:stack-id` - Stack id -3. `aws:cloudformation:logical-id` - Logical id defined in CF template +* [AWS Principal](https://gist.github.com/shortjared/4c1e3fe52bdfa47522cfe5b41e5d6f22) that are able to assume roles +* IAM Group +* IAM Group to policy relationship +* IAM Policy +* IAM Role +* IAM Role to policy relationship +* IAM User +* IAM User to group relationship +* IAM User to policy relationship + +Some roles can be aggregated to simplify the diagram. If a role is associated with a principal and is not attached to any named policy, will be aggregated. + +### AWS IoT + +Example of a diagram: + +![diagrams logo](docs/assets/aws-iot.png) + +Following resources are checked in IoT command: + +* IoT Billing Group +* IoT Certificates +* IoT Jobs +* IoT Policies +* IoT Thing +* IoT Thing Type + +### AWS All + +List all AWS resources (preview) + +The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). + +The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). + +Types of resources will mostly cover Terraform types. ### Limits usage diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index d923011..a2fc627 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -31,7 +31,7 @@ from provider.vpc.command import Vpc from provider.iot.command import Iot from provider.all.command import All -from provider.limits.command import Limits +from provider.limit.command import Limit # Check version from shared.common import ( @@ -84,7 +84,7 @@ def generate_parser(): add_default_arguments(all_parser, diagram_enabled=False) limit_parser = subparsers.add_parser( - "aws-limits", help="Analyze aws limit resources." + "aws-limit", help="Analyze aws limit resources." ) add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False) limit_parser.add_argument( @@ -228,8 +228,8 @@ def main(): ) elif args.command == "aws-all": command = All(region_names=region_names, session=session, filters=filters,) - elif args.command == "aws-limits": - command = Limits( + elif args.command == "aws-limit": + command = Limit( region_names=region_names, session=session, services=args.services, ) else: diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 83be1d6..c62e9de 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -26,7 +26,7 @@ "aws_ec2_reserved_instances_offering", "aws_ec2_snapshot", "aws_ec2_spot_price_history", - "aws_ssm_available_patche", + "aws_ssm_available_patch", "aws_ssm_document", "aws_polly_voice", "aws_lightsail_blueprint", @@ -81,6 +81,11 @@ "aws_rekognition_project", "aws_rekognition_stream_processor", "aws_sdb_domain", + "aws_redshift_table_restore_status", + "aws_iot_v2_logging_level", + "aws_license_manager_resource_inventory", + "aws_license_manager_license_configuration", + "aws_logs_query_definition", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS @@ -97,6 +102,11 @@ "GetDeploymentTarget": ["deploymentId"], "ListDeploymentTargets": ["deploymentId"], }, + "ecs": { + "ListTasks": ["cluster"], + "ListServices": ["cluster"], + "ListContainerInstances": ["cluster"], + }, "elasticbeanstalk": { "DescribeEnvironmentHealth": ["environmentName"], "DescribeEnvironmentManagedActionHistory": ["environmentName"], @@ -109,6 +119,8 @@ "ListAccessKeys": ["userName"], "ListServiceSpecificCredentials": ["userName"], "ListSigningCertificates": ["userName"], + "ListMFADevices": ["userName"], + "ListSSHPublicKeys": ["userName"], }, "iot": {"ListAuditFindings": ["taskId"]}, "opsworks": { @@ -127,6 +139,7 @@ "DescribeVolumes": ["stackId"], }, "ssm": {"DescribeMaintenanceWindowSchedule": ["windowId"],}, + "shield": {"DescribeProtection": ["protectionId"],}, "waf": { "ListActivatedRulesInRuleGroup": ["ruleGroupId"], "ListLoggingConfigurations": ["limit"], @@ -146,7 +159,11 @@ "ssm:GetParametersByPath", ] -PARALLEL_SERVICE_CALLS = 1 +SKIPPED_SERVICES = [ + "sagemaker" +] # those services have too unreliable API to make use of it + +PARALLEL_SERVICE_CALLS = 80 def _to_snake_case(camel_case): @@ -174,11 +191,27 @@ def _to_snake_case(camel_case): ) +PLURAL_TO_SINGULAR = { + "ies": "y", + "status": "status", + "ches": "ch", + "ses": "s", +} + + +def singular_from_plural(name: str) -> str: + if name.endswith("s"): + for plural_suffix, singular_suffix in PLURAL_TO_SINGULAR.items(): + if name.endswith(plural_suffix): + name = name[: -len(plural_suffix)] + singular_suffix + return name + name = name[:-1] + return name + + def last_singular_name_element(operation_name): last_name = re.findall("[A-Z][^A-Z]*", operation_name)[-1] - if last_name.endswith("s"): - last_name = last_name[:-1] - return last_name + return singular_from_plural(last_name) def retrieve_resource_name(resource, operation_name): @@ -226,9 +259,6 @@ def retrieve_resource_id(resource, operation_name, resource_name): resource_id = resource[last_name + "Arn"] elif only_one_suffix(resource, "arn"): resource_id = only_one_suffix(resource, "arn") - # type 'aws_ec2_dhcp_option' - # 'DhcpOptionsId' -> 'dopt-042d18a4769f7b35b' - # also got 'OwnerId' return resource_id @@ -281,6 +311,8 @@ def wrapper(*args, **kwargs): if ( "is not subscribed to AWS Security Hub" in exception_str or "not enabled for securityhub" in exception_str + or "The subscription does not exist" in exception_str + or "not currently delegated by AWS FM" in exception_str ): message_handler( "Operation {} not accessible, AWS Security Hub is not configured... Skipping".format( @@ -367,8 +399,11 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): message_handler( "Collecting data from {}...".format(service_full_name), "HEADER" ) - if not self.availabilityCheck.is_service_available( - self.options.region_name, aws_service + if ( + not self.availabilityCheck.is_service_available( + self.options.region_name, aws_service + ) + or aws_service in SKIPPED_SERVICES ): message_handler( "Service {} not available in this region... Skipping".format( @@ -394,15 +429,14 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): ): continue resource_type = "aws_{}_{}".format( - aws_service, + aws_service.replace("-", "_"), _to_snake_case( name.replace("List", "") .replace("Get", "") .replace("Describe", "") ), ) - if resource_type.endswith("s"): - resource_type = resource_type[:-1] + resource_type = singular_from_plural(resource_type) if resource_type in OMITTED_RESOURCES: continue if not operation_allowed(allowed_actions, aws_service, name): @@ -420,9 +454,15 @@ def analyze_operation( self, resource_type, operation_name, has_paginator, client, service_full_name ) -> List[Resource]: resources = [] + snake_operation_name = _to_snake_case(operation_name) if has_paginator: - paginator = client.get_paginator(_to_snake_case(operation_name)) - pages = paginator.paginate() + paginator = client.get_paginator(snake_operation_name) + if resource_type == "aws_iam_policy": + pages = paginator.paginate( + Scope="Local" + ) # hack to list only local IAM policies + else: + pages = paginator.paginate() list_metadata = pages.result_keys[0].parsed result_key = None result_parent = None @@ -435,7 +475,7 @@ def analyze_operation( else: message_handler( "Operation {} has unsupported pagination definition... Skipping".format( - operation_name + snake_operation_name ), "WARNING", ) @@ -454,7 +494,8 @@ def analyze_operation( if resource is not None: resources.append(resource) else: - response = getattr(client, _to_snake_case(operation_name))() + + response = getattr(client, snake_operation_name)() for response_elem in response.values(): if isinstance(response_elem, list): for response_resource in response_elem: diff --git a/cloudiscovery/provider/limits/__init__.py b/cloudiscovery/provider/limit/__init__.py similarity index 100% rename from cloudiscovery/provider/limits/__init__.py rename to cloudiscovery/provider/limit/__init__.py diff --git a/cloudiscovery/provider/limits/command.py b/cloudiscovery/provider/limit/command.py similarity index 71% rename from cloudiscovery/provider/limits/command.py rename to cloudiscovery/provider/limit/command.py index 81458c2..214350c 100644 --- a/cloudiscovery/provider/limits/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -1,10 +1,28 @@ +from typing import List + from shared.command import BaseCommand, CommandRunner from shared.common import BaseAwsOptions from shared.diagram import NoDiagram from shared.common_aws import ALLOWED_SERVICES_CODES -class Limits(BaseCommand): +class LimitOptions(BaseAwsOptions): + services: List[str] + + def __new__(cls, session, region_name, services): + """ + Limit Options + + :param session: + :param region_name: + :param services: + """ + self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) + self.services = services + return self + + +class Limit(BaseCommand): def __init__(self, region_names, session, services): """ All AWS resources @@ -25,16 +43,16 @@ def run(self): for region in self.region_names: self.init_globalaws_limits_cache(region=region, services=self.services) - limit_options = BaseAwsOptions( + limit_options = LimitOptions( session=self.session, region_name=region, services=self.services ) command_runner = CommandRunner(services=self.services) command_runner.run( - provider="limits", + provider="limit", options=limit_options, diagram_builder=NoDiagram(), title="AWS Limits - Region {}".format(region), # pylint: disable=no-member - filename=limit_options.resulting_file_name("limits"), + filename=limit_options.resulting_file_name("limit"), ) diff --git a/cloudiscovery/provider/limits/resource/__init__.py b/cloudiscovery/provider/limit/resource/__init__.py similarity index 100% rename from cloudiscovery/provider/limits/resource/__init__.py rename to cloudiscovery/provider/limit/resource/__init__.py diff --git a/cloudiscovery/provider/limits/resource/all.py b/cloudiscovery/provider/limit/resource/all.py similarity index 100% rename from cloudiscovery/provider/limits/resource/all.py rename to cloudiscovery/provider/limit/resource/all.py diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 70dc2f2..7775448 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -37,7 +37,6 @@ class bcolors: class BaseAwsOptions(NamedTuple): session: boto3.Session region_name: str - services: str def client(self, service_name: str): return self.session.client(service_name, region_name=self.region_name) diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 441d35e..bae3543 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -2,7 +2,7 @@ from cachetools import TTLCache from shared.common import ResourceCache, message_handler -from provider.limits.resource.all import ALLOWED_SERVICES_CODES +from provider.limit.resource.all import ALLOWED_SERVICES_CODES SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) @@ -37,9 +37,9 @@ def __init__(self, session, region: str, services): def init_globalaws_limits_cache(self): """ - AWS has global limits that can be adjustable and others that can't be adjustable - This method make cache for 15 days for aws cache global parameters. AWS don't update limits every time. - Services has differents limits, depending on region. + AWS has global limit that can be adjustable and others that can't be adjustable + This method make cache for 15 days for aws cache global parameters. AWS don't update limit every time. + Services has differents limit, depending on region. """ for service_code in self.services: if service_code in ALLOWED_SERVICES_CODES: @@ -50,7 +50,7 @@ def init_globalaws_limits_cache(self): continue message_handler( - "Fetching aws global limits to service {} in region {} to cache...".format( + "Fetching aws global limit to service {} in region {} to cache...".format( service_code, self.region ), "HEADER", diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 82154fe..c228cd1 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -26,7 +26,7 @@ def general_report( message_handler("\n\nFound resources", "HEADER") for resource in resources: - # Report to limits + # Report to limit if resource.limits: usage = ( str(resource.limits.usage) From 3d7f824e9fe110a47bae234727a48f4b88260b75 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 00:51:11 +0700 Subject: [PATCH 53/86] command readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 41a6aef..70e81b7 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,6 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a 2. `aws:cloudformation:stack-id` - Stack id 3. `aws:cloudformation:logical-id` - Logical id defined in CF template - ### AWS VPC Example of a diagram: From 13aae76b9eb9cd172678d23b24b0f13bcf5be56f Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 00:55:39 +0700 Subject: [PATCH 54/86] adjusted further readme --- README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 70e81b7..6a29b02 100644 --- a/README.md +++ b/README.md @@ -32,42 +32,56 @@ The commands generate reports that can be used to analyze command reports. 1. Run the cloudiscovery command with following options (if a region not informed, this script will try to get from ~/.aws/credentials): -1.1 To detect AWS VPC resources: +1.1 To detect AWS VPC resources (more on [AWS VPC](#aws-vpc)): ```sh cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] ``` -1.2 To detect AWS policy resources: +1.2 To detect AWS policy resources (more on [AWS Policy](#aws-policy)): ```sh cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] ``` -1.3 To detect AWS IoT resources: +1.3 To detect AWS IoT resources (more on [AWS IoT](#aws-iot)): ```sh cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] ``` -1.4 To detect all AWS resources: +1.4 To detect all AWS resources (more on [AWS All](#aws-all)): ```sh cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] ``` -1.5 To check AWS limits per resource: +1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): ```sh cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] ``` -Check [limits usage](#limits-usage) section. - 2. For help use: ```sh cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all|aws-limit] -h ``` +### Filtering + +It's possible to filter resources by tags and resource type. To filter, add an option `--filter `, where `` can be: + +1. `Name=tags.costCenter;Value=20000` - to filter resources by a tag name `costCenter` and with value `20000`. +2. `Name=type;Value=aws_lambda_function` to only list lambda functions. + +It's possible to pass multiple values, to be able to select a value from a set. Values are split by `:` sign. If a desired value has a `:` sign, wrap it in `'` signs e.g. `--filter="Name=tags.costCenter;Value=20000:'20001:1'`. + +It is possible to pass multiple filter options, just pass `-f filter_1 -f filter_2`. In that case, the tool will return resources that match either of the filters + +Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-aws-cloudformation-stack/): +1. `aws:cloudformation:stack-name` - Stack name +2. `aws:cloudformation:stack-id` - Stack id +3. `aws:cloudformation:logical-id` - Logical id defined in CF template + ## Requirements and Installation ### AWS Resources @@ -150,21 +164,7 @@ aws configure * (Optional) If you want to be able to switch between multiple AWS credentials and settings, you can configure [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) and later pass profile name when running the tool. -### Filtering - -It's possible to filter resources by tags and resource type. To filter, add an option `--filter `, where `` can be: - -1. `Name=tags.costCenter;Value=20000` - to filter resources by a tag name `costCenter` and with value `20000`. -2. `Name=type;Value=aws_lambda_function` to only list lambda functions. - -It's possible to pass multiple values, to be able to select a value from a set. Values are split by `:` sign. If a desired value has a `:` sign, wrap it in `'` signs e.g. `--filter="Name=tags.costCenter;Value=20000:'20001:1'`. - -It is possible to pass multiple filter options, just pass `-f filter_1 -f filter_2`. In that case, the tool will return resources that match either of the filters - -Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-aws-cloudformation-stack/): -1. `aws:cloudformation:stack-name` - Stack name -2. `aws:cloudformation:stack-id` - Stack id -3. `aws:cloudformation:logical-id` - Logical id defined in CF template +## Commands ### AWS VPC @@ -267,7 +267,7 @@ The operations must be allowed to be called by permissions described in [AWS Per Types of resources will mostly cover Terraform types. -### Limits usage +### AWS Limit It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. With `--services value,value,value` selection, you can narrow down checks to services that you want to check. From cbb72d97b760904db03b6ea55b028da1bf8ec6f1 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 19:47:38 +0100 Subject: [PATCH 55/86] Add alias to boto3 resource and new checks --- cloudiscovery/provider/limit/resource/all.py | 21 +++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 6a3f956..d5ad4c9 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -57,6 +57,14 @@ "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, "global": False, }, + "elasticfilesystem": { + "L-848C634D": { + "method": "describe_file_systems", + "key": "FileSystems", + "fields": [], + }, + "global": False, + }, "elasticbeanstalk": { "L-8EFC1C51": { "method": "describe_environments", @@ -144,6 +152,11 @@ }, } +SERVICEQUOTA_TO_BOTO3 = { + "elasticloadbalancing": "elbv2", + "elasticfilesystem": "efs", +} + class LimitResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): @@ -199,11 +212,9 @@ def get_resources(self) -> List[Resource]: "HEADER", ) - """ - TODO: Add this as alias to convert service name. - """ - if service == "elasticloadbalancing": - service = "elbv2" + # Need to convert some quota-services endpoint + if service in SERVICEQUOTA_TO_BOTO3: + service = SERVICEQUOTA_TO_BOTO3.get(service) client = self.options.session.client( service, region_name=self.options.region_name From 2053700ac501fb1c3fed2f1d088e53c95e828274 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 20:07:48 +0100 Subject: [PATCH 56/86] Fixed errors with account with no AWS organization configured --- cloudiscovery/provider/all/resource/all.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index c62e9de..31f1864 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -331,6 +331,26 @@ def wrapper(*args, **kwargs): ), "WARNING", ) + elif ( + "Your account is not a member of an organization" in exception_str + or "This action can only be made by accounts in an AWS Organization" + in exception_str + or "The request failed because organization is not in use" + in exception_str + ): + message_handler( + "Service {} only available to account in an AWS Organization... Skipping".format( + args[5] + ), + "WARNING", + ) + elif "is no longer available to new customers" in exception_str: + message_handler( + "Service {} is no longer available to new customers... Skipping".format( + args[5] + ), + "WARNING", + ) else: log_critical( "\nError running operation {}, type {}. Error message {}".format( From 0689e5c50549f6738fc5e31f0614425a18b425fd Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 21:08:06 +0100 Subject: [PATCH 57/86] Added paginator global function --- cloudiscovery/provider/all/resource/all.py | 13 +++++------ cloudiscovery/provider/limit/resource/all.py | 23 +++++++++++++++----- cloudiscovery/shared/common.py | 19 ++++++++++++++++ 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 31f1864..df9bd12 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -14,6 +14,7 @@ message_handler, ResourceAvailable, log_critical, + get_paginator, ) OMITTED_RESOURCES = [ @@ -476,13 +477,11 @@ def analyze_operation( resources = [] snake_operation_name = _to_snake_case(operation_name) if has_paginator: - paginator = client.get_paginator(snake_operation_name) - if resource_type == "aws_iam_policy": - pages = paginator.paginate( - Scope="Local" - ) # hack to list only local IAM policies - else: - pages = paginator.paginate() + pages = get_paginator( + client=client, + operation_name=snake_operation_name, + resource_type=resource_type, + ) list_metadata = pages.result_keys[0].parsed result_key = None result_parent = None diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index d5ad4c9..9edad91 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -1,5 +1,7 @@ from typing import List +from shared.common import get_paginator + from shared.common import ( ResourceProvider, Resource, @@ -37,12 +39,12 @@ "global": False, }, "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": []}, - "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": []}, + "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": [],}, + "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, "global": False, }, "dynamodb": { - "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": []}, + "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, "global": False, }, "ec2": { @@ -98,6 +100,7 @@ "method": "list_instance_profiles", "key": "InstanceProfiles", "fields": [], + "paginate": False, }, "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, "global": True, @@ -220,9 +223,19 @@ def get_resources(self) -> List[Resource]: service, region_name=self.options.region_name ) - response = getattr(client, quota_data["method"])() + usage = 0 - usage = len(response[quota_data["key"]]) + pages = get_paginator( + client=client, + operation_name=quota_data["method"], + resource_type="aws_limit", + ) + if not pages: + response = getattr(client, quota_data["method"])() + usage = len(response[quota_data["key"]]) + else: + for page in pages: + usage = usage + len(page[quota_data["key"]]) percent = round((usage / value) * 100, 2) diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 7775448..864a3fe 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -320,3 +320,22 @@ def parse_filters(arg_filters) -> List[Filterable]: _add_filter(filters, is_tag, full_name, "".join(val_buffer)) return filters + + +def get_paginator(client, operation_name, resource_type): + """ + TODO: Possible circular reference using in common_aws, move to there in future. + """ + # Checking if can page + if client.can_paginate(operation_name): + paginator = client.get_paginator(operation_name) + if resource_type == "aws_iam_policy": + pages = paginator.paginate( + Scope="Local" + ) # hack to list only local IAM policies - aws_all + else: + pages = paginator.paginate() + else: + return False + + return pages From 995180a6c7ec8e989390e6f3cfa3776078fff158 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 22:57:58 +0100 Subject: [PATCH 58/86] Added new checks --- cloudiscovery/provider/limit/resource/all.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 9edad91..8d79208 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -103,6 +103,11 @@ "paginate": False, }, "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, + "L-DB618D39": { + "method": "list_saml_providers", + "key": "SAMLProviderList", + "fields": [], + }, "global": True, }, "kms": { @@ -122,6 +127,19 @@ }, "global": True, }, + "route53resolver": { + "L-4A669CC0": { + "method": "list_resolver_endpoints", + "key": "ResolverEndpoints", + "fields": [], + }, + "L-51D8A1FB": { + "method": "list_resolver_rules", + "key": "ResolverRules", + "fields": [], + }, + "global": True, + }, "rds": { "L-7B6409FD": { "method": "describe_db_instances", From 63bacf8d2faf47ba92ba805f24738df317109ac2 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 23:23:55 +0100 Subject: [PATCH 59/86] Added new checks --- cloudiscovery/provider/limit/resource/all.py | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 8d79208..f821487 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -114,6 +114,28 @@ "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, "global": False, }, + "mediaconnect": { + "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, + "L-F1F62F5D": { + "method": "list_entitlements", + "key": "Entitlements", + "fields": [], + }, + "global": False, + }, + "medialive": { + "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, + "L-BDF24E14": { + "method": "list_input_devices", + "key": "InputDevices", + "fields": [], + }, + "global": False, + }, + "mediapackage": { + "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, + "global": False, + }, "route53": { "L-4EA4796A": { "method": "list_hosted_zones", From dee598ab00eaa7f9ece74428f7669e676385980f Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 23:36:03 +0100 Subject: [PATCH 60/86] Added new checks --- README.md | 8 ++++++++ cloudiscovery/provider/limit/resource/all.py | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/README.md b/README.md index 6a29b02..8255d97 100644 --- a/README.md +++ b/README.md @@ -280,10 +280,18 @@ It's possible to check resources limits in an account. This script allows check * dynamodb * ec2 * ecs + * elasticfilesystem * elasticbeanstalk * elasticloadbalancing * iam + * kms + * mediaconnect + * medialive + * mediapackage + * qldb + * robomaker * route53 + * route53resolver * rds * s3 * sns diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index f821487..5d3d49f 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -136,6 +136,19 @@ "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, "global": False, }, + "qldb": { + "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, + "global": False, + }, + "robomaker": { + "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, + "L-D6554FB1": { + "method": "list_simulation_applications", + "key": "simulationApplicationSummaries", + "fields": [], + }, + "global": False, + }, "route53": { "L-4EA4796A": { "method": "list_hosted_zones", From 9e3ab81255693cb727da02cf627daaf08f95190f Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 29 Jun 2020 23:37:01 +0100 Subject: [PATCH 61/86] Comment issue --- cloudiscovery/shared/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 864a3fe..8e54a27 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -326,7 +326,7 @@ def get_paginator(client, operation_name, resource_type): """ TODO: Possible circular reference using in common_aws, move to there in future. """ - # Checking if can page + # Checking if can paginate if client.can_paginate(operation_name): paginator = client.get_paginator(operation_name) if resource_type == "aws_iam_policy": From 49efa0bbab908a905b042f1cd4132f486f042431 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 30 Jun 2020 00:41:34 +0100 Subject: [PATCH 62/86] New resources --- README.md | 2 ++ cloudiscovery/provider/limit/resource/all.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/README.md b/README.md index 8255d97..0b82624 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,8 @@ It's possible to check resources limits in an account. This script allows check * rds * s3 * sns + * transcribe + * translate AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. An administrator can ask to increase the quota value of a certain service via ticket and this script will detect this. diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 5d3d49f..a592186 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -206,6 +206,22 @@ "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, "global": False, }, + "transcribe": { + "L-3278D334": { + "method": "list_vocabularies", + "key": "Vocabularies", + "fields": [], + }, + "global": False, + }, + "translate": { + "L-4011ABD8": { + "method": "list_terminologies", + "key": "TerminologyPropertiesList", + "fields": [], + }, + "global": False, + }, } SERVICEQUOTA_TO_BOTO3 = { From f5cbf166ce5d2ae3302a96dd59a2b3a7c5c4c444 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 30 Jun 2020 10:44:33 +0100 Subject: [PATCH 63/86] Added parallel execution and usage limit --- .prospector.yaml | 2 +- cloudiscovery/__init__.py | 22 ++- cloudiscovery/provider/limit/command.py | 12 +- cloudiscovery/provider/limit/resource/all.py | 136 ++++++++++++------- cloudiscovery/shared/report.py | 12 +- 5 files changed, 123 insertions(+), 61 deletions(-) diff --git a/.prospector.yaml b/.prospector.yaml index 9dbfd96..85b9d62 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -20,4 +20,4 @@ pep8: mccabe: options: - max-complexity: 19 \ No newline at end of file + max-complexity: 21 \ No newline at end of file diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index a2fc627..3738a41 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -94,6 +94,13 @@ def generate_parser(): help='Inform services that you want to check, use "," (comma) to split them. \ If not informed, script will check all services.', ) + limit_parser.add_argument( + "-u", + "--usage", + required=False, + help="Inform the %% of resource usage between 0 and 100. \ + For example: --usage 50 will get all resources with more than 50%% usage.", + ) return parser @@ -135,7 +142,7 @@ def add_default_arguments( ) -# pylint: disable=too-many-branches +# pylint: disable=too-many-branches,too-many-statements def main(): # Entry point for the CLI. # Load commands @@ -203,6 +210,14 @@ def main(): region_parameter=args.region_name, region_name=region_name, session=session, ) + if "usage" in args: + if args.usage is not None: + if args.usage.isdigit() is False: + exit_critical(_("Usage must be between 0 and 100")) + else: + if int(args.usage) < 0 or int(args.usage) > 100: + exit_critical(_("Usage must be between 0 and 100")) + if args.command == "aws-vpc": command = Vpc( vpc_id=args.vpc_id, @@ -230,7 +245,10 @@ def main(): command = All(region_names=region_names, session=session, filters=filters,) elif args.command == "aws-limit": command = Limit( - region_names=region_names, session=session, services=args.services, + region_names=region_names, + session=session, + services=args.services, + usage=args.usage, ) else: raise NotImplementedError("Unknown command") diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 214350c..22330a6 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -9,7 +9,7 @@ class LimitOptions(BaseAwsOptions): services: List[str] - def __new__(cls, session, region_name, services): + def __new__(cls, session, region_name, services, usage): """ Limit Options @@ -19,11 +19,12 @@ def __new__(cls, session, region_name, services): """ self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) self.services = services + self.usage = usage return self class Limit(BaseCommand): - def __init__(self, region_names, session, services): + def __init__(self, region_names, session, services, usage): """ All AWS resources @@ -39,12 +40,17 @@ def __init__(self, region_names, session, services): else: self.services = services.split(",") + self.usage = usage + def run(self): for region in self.region_names: self.init_globalaws_limits_cache(region=region, services=self.services) limit_options = LimitOptions( - session=self.session, region_name=region, services=self.services + session=self.session, + region_name=region, + services=self.services, + usage=self.usage, ) command_runner = CommandRunner(services=self.services) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index a592186..90150c3 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -1,5 +1,7 @@ from typing import List +from concurrent.futures.thread import ThreadPoolExecutor + from shared.common import get_paginator from shared.common import ( @@ -229,6 +231,8 @@ "elasticfilesystem": "efs", } +MAX_EXECUTION_PARALLEL = 4 + class LimitResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): @@ -245,69 +249,103 @@ def __init__(self, options: BaseAwsOptions): # pylint: disable=too-many-locals def get_resources(self) -> List[Resource]: + usage_check = 0 if self.options.usage is None else self.options.usage + client_quota = self.options.session.client("service-quotas") resources_found = [] services = self.options.services - for service in services: - cache_key = "aws_limits_" + service + "_" + self.options.region_name - cache = self.cache.get_key(cache_key) - - for data_quota_code in cache[service]: - quota_data = ALLOWED_SERVICES_CODES[service][ - data_quota_code["quota_code"] - ] - - value_aws = data_quota_code["value"] - - # Quota is adjustable by ticket request, then must override this values - if bool(data_quota_code["adjustable"]) is True: - try: - response_quota = client_quota.get_service_quota( - ServiceCode=service, QuotaCode=data_quota_code["quota_code"] - ) - if "Value" in response_quota["Quota"]: - value = response_quota["Quota"]["Value"] - else: - value = data_quota_code["value"] - except client_quota.exceptions.NoSuchResourceException: + with ThreadPoolExecutor(MAX_EXECUTION_PARALLEL) as executor: + results = executor.map( + lambda aws_limit: self.analyze_service( + aws_limit=aws_limit, + client_quota=client_quota, + usage_check=int(usage_check), + ), + services, + ) + + for result in results: + resources_found.extend(result) + + return resources_found + + @exception + def analyze_service(self, aws_limit, client_quota, usage_check): + + service = aws_limit + + cache_key = "aws_limits_" + service + "_" + self.options.region_name + cache = self.cache.get_key(cache_key) + + return self.analyze_detail( + client_quota=client_quota, + data_resource=cache[service], + service=service, + usage_check=usage_check, + ) + + @exception + # pylint: disable=too-many-locals + def analyze_detail(self, client_quota, data_resource, service, usage_check): + + resources_found = [] + + for data_quota_code in data_resource: + + quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] + + value_aws = value = data_quota_code["value"] + + # Quota is adjustable by ticket request, then must override this values + if bool(data_quota_code["adjustable"]) is True: + try: + response_quota = client_quota.get_service_quota( + ServiceCode=service, QuotaCode=data_quota_code["quota_code"] + ) + if "Value" in response_quota["Quota"]: + value = response_quota["Quota"]["Value"] + else: value = data_quota_code["value"] + except client_quota.exceptions.NoSuchResourceException: + value = data_quota_code["value"] - message_handler( - "Collecting data from Quota: " - + service - + " - " - + data_quota_code["quota_name"] - + "...", - "HEADER", - ) + message_handler( + "Collecting data from Quota: " + + service + + " - " + + data_quota_code["quota_name"] + + "...", + "HEADER", + ) - # Need to convert some quota-services endpoint - if service in SERVICEQUOTA_TO_BOTO3: - service = SERVICEQUOTA_TO_BOTO3.get(service) + # Need to convert some quota-services endpoint + if service in SERVICEQUOTA_TO_BOTO3: + service = SERVICEQUOTA_TO_BOTO3.get(service) - client = self.options.session.client( - service, region_name=self.options.region_name - ) + client = self.options.session.client( + service, region_name=self.options.region_name + ) - usage = 0 + usage = 0 - pages = get_paginator( - client=client, - operation_name=quota_data["method"], - resource_type="aws_limit", - ) - if not pages: - response = getattr(client, quota_data["method"])() - usage = len(response[quota_data["key"]]) - else: - for page in pages: - usage = usage + len(page[quota_data["key"]]) + pages = get_paginator( + client=client, + operation_name=quota_data["method"], + resource_type="aws_limit", + ) + if not pages: + response = getattr(client, quota_data["method"])() + usage = len(response[quota_data["key"]]) + else: + for page in pages: + usage = usage + len(page[quota_data["key"]]) - percent = round((usage / value) * 100, 2) + percent = round((usage / value) * 100, 2) + if percent >= usage_check: resources_found.append( Resource( digest=ResourceDigest( diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index c228cd1..9030ab9 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -101,12 +101,12 @@ def html_report( diagram_image=diagram_image, ) - self.make_directories() + self.make_directories() - name_output = PATH_REPORT_HTML_OUTPUT + filename + ".html" + name_output = PATH_REPORT_HTML_OUTPUT + filename + ".html" - with open(name_output, "w") as file_output: - file_output.write(html_output) + with open(name_output, "w") as file_output: + file_output.write(html_output) - message_handler("\n\nHTML report generated", "HEADER") - message_handler("Check your HTML report: " + name_output, "OKBLUE") + message_handler("\n\nHTML report generated", "HEADER") + message_handler("Check your HTML report: " + name_output, "OKBLUE") From 3f41101526c6cb3aa35d6d35ffbd712af1b3c1e4 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 30 Jun 2020 10:50:32 +0100 Subject: [PATCH 64/86] Readme update --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0b82624..93bdabc 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filt 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): ```sh -cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] +cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--usage 0-100] ``` 2. For help use: @@ -269,7 +269,7 @@ Types of resources will mostly cover Terraform types. ### AWS Limit -It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. With `--services value,value,value` selection, you can narrow down checks to services that you want to check. +It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. With `--services value,value,value` selection, you can narrow down checks to services that you want to check. With `--usage 0-100` selection, you can inform a minimum usage % of a certain service. * Services available * acm From ff26927e0e7376ae9942df835699dbcfdf590e99 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 30 Jun 2020 15:06:34 +0100 Subject: [PATCH 65/86] Codacy fix --- cloudiscovery/provider/limit/resource/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 90150c3..70c652e 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -299,7 +299,7 @@ def analyze_detail(self, client_quota, data_resource, service, usage_check): value_aws = value = data_quota_code["value"] - # Quota is adjustable by ticket request, then must override this values + # Quota is adjustable by ticket request, then must override this values. if bool(data_quota_code["adjustable"]) is True: try: response_quota = client_quota.get_service_quota( From afc0ea52d97419b0c98798a6c1bb6b5c3cea9331 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 22:51:49 +0700 Subject: [PATCH 66/86] small adjustments to all command --- README.md | 36 ++++++------ cloudiscovery/provider/all/resource/all.py | 56 +++++++++++++++---- cloudiscovery/shared/command.py | 2 +- .../tests/provider/all/resource/test_all.py | 6 ++ 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0b82624..79e77e2 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The tool consists of various commands to help you understand the cloud infrastru ### Diagrams -Commands can generate diagrams. When modelling them, we try to follow the following principle: +Commands can generate diagrams. When modelling them, we try to follow the principle: > Graphical excellence is that which gives to the viewer the greatest number of ideas in the shortest time with the least ink in the smallest space. @@ -26,11 +26,11 @@ Edward Tufte ## Report -The commands generate reports that can be used to analyze command reports. +The commands generate reports that can be used to further analyze resources. ### CLI -1. Run the cloudiscovery command with following options (if a region not informed, this script will try to get from ~/.aws/credentials): +1. Run the cloudiscovery command with following options (if a region not pass, this script will try to get it from ~/.aws/credentials): 1.1 To detect AWS VPC resources (more on [AWS VPC](#aws-vpc)): @@ -84,25 +84,29 @@ Useful [CF tags](https://aws.amazon.com/blogs/devops/tracking-the-cost-of-your-a ## Requirements and Installation -### AWS Resources +### Installation -This script has been written in python3+ and AWS-CLI and it works in Linux, Windows and OSX. +This tool has been written in Python3+ and AWS-CLI and it works on Linux, Windows and Mac OS. -* Make sure the latest version of AWS-CLI is installed on your workstation, and other components needed, with Python pip already installed: +Make sure the latest version of AWS-CLI is installed on your workstation, and other components needed, with Python pip already installed: ```sh pip install -U cloudiscovery ``` -* Make sure you have properly configured your AWS-CLI with a valid Access Key and Region: +### AWS Credentials + +Make sure you have properly configured your AWS-CLI with a valid Access Key and Region: ```sh aws configure ``` -### AWS Permissions +More on credentials configuration: [Configuration basics](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) + +#### AWS Permissions -* The configured credentials must be associated to a user or role with proper permissions to do all checks. If you want to use a role with narrowed set of permissions just to perform cloud discovery, use a role from the following CF template shown below. To further increase security, you can add a block to check `aws:MultiFactorAuthPresent` condition in `AssumeRolePolicyDocument`. More on using IAM roles in the [configuration file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html). +The configured credentials must be associated to a user or role with proper permissions to do all checks. If you want to use a role with narrowed set of permissions just to perform cloud discovery, use a role from the following CF template shown below. To further increase security, you can add a block to check `aws:MultiFactorAuthPresent` condition in `AssumeRolePolicyDocument`. More on using IAM roles in the [configuration file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html). ```json { @@ -162,7 +166,7 @@ aws configure } ``` -* (Optional) If you want to be able to switch between multiple AWS credentials and settings, you can configure [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) and later pass profile name when running the tool. +(Optional) If you want to be able to switch between multiple AWS credentials and settings, you can configure [named profiles](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) and later pass profile name when running the tool. ## Commands @@ -265,7 +269,7 @@ The command tries to call all AWS services (200+) and operations with name `Desc The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). -Types of resources will mostly cover Terraform types. +Types of resources mostly cover Terraform types. ### AWS Limit @@ -301,7 +305,7 @@ It's possible to check resources limits in an account. This script allows check AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. An administrator can ask to increase the quota value of a certain service via ticket and this script will detect this. -### Using a Docker container +## Using a Docker container To build docker container using Dockerfile @@ -322,7 +326,7 @@ cloudiscovery \ * If you are using Diagram output and due to fact container is a slim image of Python image, you must run cloudiscovery with "--diagram False", otherwise you'll have an error about "xdg-open". The output file will be saved in "assets/diagrams". -### Translate +## Translate This project support English and Portuguese (Brazil) languages. To contribute with a translation, follow this steps: @@ -333,11 +337,11 @@ This project support English and Portuguese (Brazil) languages. To contribute wi python msgfmt.py -o locales/NEWFOLDER/LC_MESSAGES/messages.mo locales/NEWFOLDER/LC_MESSAGES/messages ``` -### Contributing +## Contributing If you have improvements or fixes, we would love to have your contributions. Please use [PEP 8](https://pycodestyle.readthedocs.io/en/latest/) code style. -### Development +## Development When developing, it's recommended to use [venv](https://docs.python.org/3/library/venv.html). @@ -385,7 +389,7 @@ To add new resources to check limit, please remove "assets/.cache/cache.db" 1. Update the version in cloudiscovery/__init\__.py and create a new git tag with `git tag $VERSION`. 2. Once you push the tag to GitHub with `git push --tags`, a new CircleCI build is triggered. -### Similar projects and products +## Similar projects and products * [mingrammer/diagrams](https://github.com/mingrammer/diagrams) - library being used to draw diagrams * [Lucidchart Cloud Insights](https://www.lucidchart.com/pages/solutions/cloud-insights) - commercial extension to Lucidchart diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index df9bd12..474b0b0 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -87,6 +87,23 @@ "aws_license_manager_resource_inventory", "aws_license_manager_license_configuration", "aws_logs_query_definition", + "aws_autoscaling_scaling_activity", + "aws_cloudwatch_metric", + "aws_organizations_handshakes_for_organization", + "aws_config_organization_config_rule", + "aws_organizations_root", + "aws_organizations_delegated_administrator", + "aws_organizations_create_account_status", + "aws_config_organization_conformance_pack_status", + "aws_config_organization_conformance_pack", + "aws_ec2_reserved_instances_listing", + "aws_redshift_cluster_security_group", + "aws_guardduty_organization_admin_account", + "aws_elasticache_cache_security_group", + "aws_organizations_aws_service_access_for_organization", + "aws_organizations_account", + "aws_config_organization_config_rule_status", + "aws_dynamodb_backup", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS @@ -189,6 +206,9 @@ def _to_snake_case(camel_case): .replace("dbproxies", "db_proxies") .replace("dbparameter", "db_parameter") .replace("dbinstance", "db_instance") + .replace("d_bparameter", "db_parameter") + .replace("s_amlproviders", "saml_providers") + .replace("a_wsservice", "aws_service") ) @@ -206,7 +226,8 @@ def singular_from_plural(name: str) -> str: if name.endswith(plural_suffix): name = name[: -len(plural_suffix)] + singular_suffix return name - name = name[:-1] + if not name.endswith("ss"): + name = name[:-1] return name @@ -313,7 +334,7 @@ def wrapper(*args, **kwargs): "is not subscribed to AWS Security Hub" in exception_str or "not enabled for securityhub" in exception_str or "The subscription does not exist" in exception_str - or "not currently delegated by AWS FM" in exception_str + or "calling the DescribeHub operation" in exception_str ): message_handler( "Operation {} not accessible, AWS Security Hub is not configured... Skipping".format( @@ -352,6 +373,16 @@ def wrapper(*args, **kwargs): ), "WARNING", ) + elif ( + "only available to Master account in AWS FM" in exception_str + or "not currently delegated by AWS FM" in exception_str + ): + message_handler( + "Operation {} not accessible, not master account in AWS FM... Skipping".format( + args[2] + ), + "WARNING", + ) else: log_critical( "\nError running operation {}, type {}. Error message {}".format( @@ -368,6 +399,17 @@ def wrapper(*args, **kwargs): return wrapper +def build_resource_type(aws_service, name): + resource_name = re.sub(r"^List", "", name) + resource_name = re.sub(r"^Get", "", resource_name) + resource_name = re.sub(r"^Describe", "", resource_name) + return singular_from_plural( + "aws_{}_{}".format( + aws_service.replace("-", "_"), _to_snake_case(resource_name), + ) + ) + + class AllResources(ResourceProvider): def __init__(self, options: BaseAwsOptions): """ @@ -449,15 +491,7 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): and operation["name"] in REQUIRED_PARAMS_OVERRIDE[aws_service] ): continue - resource_type = "aws_{}_{}".format( - aws_service.replace("-", "_"), - _to_snake_case( - name.replace("List", "") - .replace("Get", "") - .replace("Describe", "") - ), - ) - resource_type = singular_from_plural(resource_type) + resource_type = build_resource_type(aws_service, name) if resource_type in OMITTED_RESOURCES: continue if not operation_allowed(allowed_actions, aws_service, name): diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index 58f0f4f..18b629a 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -108,7 +108,7 @@ def __init__(self, filters=None, services=None): :param filters: """ self.filters: List[Filterable] = filters - self.services = services + self.services: List[str] = services # pylint: disable=too-many-locals,too-many-arguments def run( diff --git a/cloudiscovery/tests/provider/all/resource/test_all.py b/cloudiscovery/tests/provider/all/resource/test_all.py index 0eed4a2..d6a4506 100644 --- a/cloudiscovery/tests/provider/all/resource/test_all.py +++ b/cloudiscovery/tests/provider/all/resource/test_all.py @@ -7,6 +7,7 @@ retrieve_resource_id, last_singular_name_element, operation_allowed, + build_resource_type, ) @@ -62,3 +63,8 @@ def test_operation_allowed(self): assert_that(operation_allowed(["ecs:List*"], "iam", "ListRoles")).is_equal_to( False ) + + def test_build_resource_type(self): + assert_that(build_resource_type("rds", "DescribeDBParameterGroup")).is_equal_to( + "aws_rds_db_parameter_group" + ) From 76800b0e54f607f03df5b98880bac2b60f8529ee Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 23:02:55 +0700 Subject: [PATCH 67/86] limits code review --- cloudiscovery/provider/limit/resource/all.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 90150c3..db684c3 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -231,7 +231,7 @@ "elasticfilesystem": "efs", } -MAX_EXECUTION_PARALLEL = 4 +MAX_EXECUTION_PARALLEL = 3 class LimitResources(ResourceProvider): @@ -268,7 +268,8 @@ def get_resources(self) -> List[Resource]: ) for result in results: - resources_found.extend(result) + if result is not None: + resources_found.extend(result) return resources_found From 83f802bbf6aa92f3e3513e9689a7e6bde108f8a8 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 30 Jun 2020 17:08:21 +0100 Subject: [PATCH 68/86] Added new resources --- README.md | 4 +++ cloudiscovery/provider/limit/resource/all.py | 28 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/README.md b/README.md index 93bdabc..0bc4d4b 100644 --- a/README.md +++ b/README.md @@ -274,6 +274,10 @@ It's possible to check resources limits in an account. This script allows check * Services available * acm * amplify + * appmesh + * appsync + * autoscaling-plans + * batch * codebuild * codecommit * cloudformation diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 70c652e..a6988f9 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -28,6 +28,34 @@ "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, "global": False, }, + "appmesh": { + "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, + "global": False, + }, + "appsync": { + "L-06A0647C": { + "method": "list_graphql_apis", + "key": "graphqlApis", + "fields": [], + }, + "global": False, + }, + "autoscaling-plans": { + "L-BD401546": { + "method": "describe_scaling_plans", + "key": "ScalingPlans", + "fields": [], + }, + "global": False, + }, + "batch": { + "L-144F0CA5": { + "method": "describe_compute_environments", + "key": "computeEnvironments", + "fields": [], + }, + "global": False, + }, "codebuild": { "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, "global": False, From 6f95f9a31af0ee8e380e5250da7b6e6980b07edc Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 23:09:47 +0700 Subject: [PATCH 69/86] limits docs adjustments --- README.md | 10 ++++++++-- cloudiscovery/__init__.py | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 93bdabc..9a2adba 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,11 @@ Types of resources will mostly cover Terraform types. ### AWS Limit -It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. With `--services value,value,value` selection, you can narrow down checks to services that you want to check. With `--usage 0-100` selection, you can inform a minimum usage % of a certain service. +It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. + +With `--services value,value,value` selection, you can narrow down checks to services that you want to check. + +With `--threshold 0-100` option, you can customize a minimum percentage threshold to start reporting a warning. * Services available * acm @@ -299,7 +303,9 @@ It's possible to check resources limits in an account. This script allows check * translate AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. -An administrator can ask to increase the quota value of a certain service via ticket and this script will detect this. +An administrator can ask to increase the quota value of a certain service via ticket. This command helps administrators detect those issues in advance. + +More information: [AWS WA, REL 1 How do you manage service limits?](https://wa.aws.amazon.com/wat.question.REL_1.en.html) ### Using a Docker container diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 3738a41..d0ae3d0 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -95,11 +95,11 @@ def generate_parser(): If not informed, script will check all services.', ) limit_parser.add_argument( - "-u", - "--usage", + "-t", + "--threshold", required=False, - help="Inform the %% of resource usage between 0 and 100. \ - For example: --usage 50 will get all resources with more than 50%% usage.", + help="Select the % of resource usage between 0 and 100. \ + For example: --threshold 50 will report all resources with more than 50% usage.", ) return parser From 86d2ae4f405073f1f279291e674c70461fdc9de3 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Tue, 30 Jun 2020 23:22:13 +0700 Subject: [PATCH 70/86] limits param adjustments --- cloudiscovery/__init__.py | 18 +++++++++--------- cloudiscovery/provider/limit/command.py | 10 +++++----- cloudiscovery/provider/limit/resource/all.py | 14 ++++++++------ 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index d0ae3d0..528f4aa 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -98,8 +98,8 @@ def generate_parser(): "-t", "--threshold", required=False, - help="Select the % of resource usage between 0 and 100. \ - For example: --threshold 50 will report all resources with more than 50% usage.", + help="Select the % of resource threshold between 0 and 100. \ + For example: --threshold 50 will report all resources with more than 50% threshold.", ) return parser @@ -210,13 +210,13 @@ def main(): region_parameter=args.region_name, region_name=region_name, session=session, ) - if "usage" in args: - if args.usage is not None: - if args.usage.isdigit() is False: - exit_critical(_("Usage must be between 0 and 100")) + if "threshold" in args: + if args.threshold is not None: + if args.threshold.isdigit() is False: + exit_critical(_("Threshold must be between 0 and 100")) else: - if int(args.usage) < 0 or int(args.usage) > 100: - exit_critical(_("Usage must be between 0 and 100")) + if int(args.threshold) < 0 or int(args.threshold) > 100: + exit_critical(_("Threshold must be between 0 and 100")) if args.command == "aws-vpc": command = Vpc( @@ -248,7 +248,7 @@ def main(): region_names=region_names, session=session, services=args.services, - usage=args.usage, + threshold=args.threshold, ) else: raise NotImplementedError("Unknown command") diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 22330a6..ecf9268 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -9,7 +9,7 @@ class LimitOptions(BaseAwsOptions): services: List[str] - def __new__(cls, session, region_name, services, usage): + def __new__(cls, session, region_name, services, threshold): """ Limit Options @@ -19,12 +19,12 @@ def __new__(cls, session, region_name, services, usage): """ self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) self.services = services - self.usage = usage + self.threshold = threshold return self class Limit(BaseCommand): - def __init__(self, region_names, session, services, usage): + def __init__(self, region_names, session, services, threshold): """ All AWS resources @@ -40,7 +40,7 @@ def __init__(self, region_names, session, services, usage): else: self.services = services.split(",") - self.usage = usage + self.threshold = threshold def run(self): @@ -50,7 +50,7 @@ def run(self): session=self.session, region_name=region, services=self.services, - usage=self.usage, + threshold=self.threshold, ) command_runner = CommandRunner(services=self.services) diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 8160f4a..3aee7a7 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -277,7 +277,9 @@ def __init__(self, options: BaseAwsOptions): # pylint: disable=too-many-locals def get_resources(self) -> List[Resource]: - usage_check = 0 if self.options.usage is None else self.options.usage + threshold_requested = ( + 0 if self.options.threshold is None else self.options.threshold + ) client_quota = self.options.session.client("service-quotas") @@ -290,7 +292,7 @@ def get_resources(self) -> List[Resource]: lambda aws_limit: self.analyze_service( aws_limit=aws_limit, client_quota=client_quota, - usage_check=int(usage_check), + threshold_requested=int(threshold_requested), ), services, ) @@ -302,7 +304,7 @@ def get_resources(self) -> List[Resource]: return resources_found @exception - def analyze_service(self, aws_limit, client_quota, usage_check): + def analyze_service(self, aws_limit, client_quota, threshold_requested): service = aws_limit @@ -313,12 +315,12 @@ def analyze_service(self, aws_limit, client_quota, usage_check): client_quota=client_quota, data_resource=cache[service], service=service, - usage_check=usage_check, + threshold_requested=threshold_requested, ) @exception # pylint: disable=too-many-locals - def analyze_detail(self, client_quota, data_resource, service, usage_check): + def analyze_detail(self, client_quota, data_resource, service, threshold_requested): resources_found = [] @@ -374,7 +376,7 @@ def analyze_detail(self, client_quota, data_resource, service, usage_check): percent = round((usage / value) * 100, 2) - if percent >= usage_check: + if percent >= threshold_requested: resources_found.append( Resource( digest=ResourceDigest( From eeb4a97c1051dcfa421a24ef4c433ef29f99a445 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 2 Jul 2020 11:11:55 +0100 Subject: [PATCH 71/86] Added verbose mode #121 --- README.md | 14 +++++++----- cloudiscovery/__init__.py | 34 +++++++++++++++++++----------- cloudiscovery/shared/common_aws.py | 10 +++++++++ 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ca6e9c6..ca50598 100644 --- a/README.md +++ b/README.md @@ -35,29 +35,29 @@ The commands generate reports that can be used to further analyze resources. 1.1 To detect AWS VPC resources (more on [AWS VPC](#aws-vpc)): ```sh -cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] +cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] ``` 1.2 To detect AWS policy resources (more on [AWS Policy](#aws-policy)): ```sh -cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] +cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] ``` 1.3 To detect AWS IoT resources (more on [AWS IoT](#aws-iot)): ```sh -cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] +cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] ``` 1.4 To detect all AWS resources (more on [AWS All](#aws-all)): ```sh -cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose DEBUG] ``` 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): ```sh -cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--usage 0-100] +cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--usage 0-100] [--verbose DEBUG] ``` 2. For help use: @@ -66,6 +66,10 @@ cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--se cloudiscovery [aws-vpc|aws-policy|aws-iot|aws-all|aws-limit] -h ``` +### Debbuging + +Enabling verbose mode, it is possible to debug all calls to the providers endpoints and check possible problems. + ### Filtering It's possible to filter resources by tags and resource type. To filter, add an option `--filter `, where `` can be: diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 528f4aa..4c9764c 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -33,16 +33,16 @@ from provider.all.command import All from provider.limit.command import Limit -# Check version from shared.common import ( exit_critical, generate_session, Filterable, parse_filters, ) +from shared.common_aws import aws_verbose # pylint: enable=wrong-import-position - +# Check version if sys.version_info < (3, 6): print("Python 3.6 or newer is required", file=sys.stderr) sys.exit(1) @@ -120,7 +120,10 @@ def add_default_arguments( "-p", "--profile-name", required=False, help="Profile to be used" ) parser.add_argument( - "-l", "--language", required=False, help="available languages: pt_BR, en_US" + "-l", "--language", required=False, help="Available languages: pt_BR, en_US" + ) + parser.add_argument( + "-ve", "--verbose", required=False, help="Enable debug mode to sdk calls" ) if filters_enabled: parser.add_argument( @@ -153,6 +156,10 @@ def main(): args = parser.parse_args() + # Check if verbose mode is enabled + if args.verbose: + aws_verbose(verbose_mode=args.verbose) + if args.language is None or args.language not in AVAILABLE_LANGUAGES: language = "en_US" else: @@ -174,15 +181,7 @@ def main(): _ = defaultlanguage.gettext # diagram version check - if diagram: - # Checking diagram version. Must be 0.13 or higher - if pkg_resources.get_distribution("diagrams").version < "0.14": - exit_critical( - _( - "You must update diagrams package to 0.14 or higher. " - "- See on https://github.com/mingrammer/diagrams" - ) - ) + check_diagram_version(diagram) # filters check if "filters" in args: @@ -255,6 +254,17 @@ def main(): command.run() +def check_diagram_version(diagram): + + if diagram: + # Checking diagram version. Must be 0.13 or higher + if pkg_resources.get_distribution("diagrams").version < "0.14": + exit_critical( + "You must update diagrams package to 0.14 or higher. " + "- See on https://github.com/mingrammer/diagrams" + ) + + def check_region(region_parameter, region_name, session): """ Region us-east-1 as a default region here diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index bae3543..cb6cbbe 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -1,4 +1,5 @@ import botocore.exceptions +import boto3 from cachetools import TTLCache from shared.common import ResourceCache, message_handler @@ -23,6 +24,15 @@ def describe_subnet(vpc_options, subnet_ids): return None +def aws_verbose(verbose_mode="DEBUG"): + """ + Boto3 only provides usable information in DEBUG mode + Using empty name it catchs debug from boto3/botocore + TODO: Open a ticket in boto3/botocore project to provide more information at other levels of debugging + """ + boto3.set_stream_logger(name="") + + class LimitParameters: def __init__(self, session, region: str, services): self.region = region From b22c33bf6d6421cc009fff2ad0f6fe5946c5f749 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Thu, 2 Jul 2020 16:17:17 +0100 Subject: [PATCH 72/86] Verbose mode standards #121 --- cloudiscovery/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 4c9764c..413daee 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -98,8 +98,8 @@ def generate_parser(): "-t", "--threshold", required=False, - help="Select the % of resource threshold between 0 and 100. \ - For example: --threshold 50 will report all resources with more than 50% threshold.", + help="Select the %% of resource threshold between 0 and 100. \ + For example: --threshold 50 will report all resources with more than 50%% threshold.", ) return parser @@ -123,7 +123,7 @@ def add_default_arguments( "-l", "--language", required=False, help="Available languages: pt_BR, en_US" ) parser.add_argument( - "-ve", "--verbose", required=False, help="Enable debug mode to sdk calls" + "--verbose", "--verbose", required=False, help="Enable debug mode to sdk calls" ) if filters_enabled: parser.add_argument( From 21fbf8de7213bb6d21e619af6340ec1eb3941ae5 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 01:15:34 +0700 Subject: [PATCH 73/86] code refactoring, verbosing logs --- README.md | 19 +- cloudiscovery/__init__.py | 76 ++-- cloudiscovery/provider/all/command.py | 37 +- cloudiscovery/provider/all/resource/all.py | 18 +- cloudiscovery/provider/iot/command.py | 56 +-- .../provider/iot/resource/certificate.py | 5 +- cloudiscovery/provider/iot/resource/policy.py | 5 +- cloudiscovery/provider/iot/resource/thing.py | 14 +- cloudiscovery/provider/limit/command.py | 381 ++++++++++++++++-- cloudiscovery/provider/limit/resource/all.py | 263 +----------- cloudiscovery/provider/policy/command.py | 34 +- .../provider/policy/resource/general.py | 9 +- .../provider/policy/resource/security.py | 26 +- cloudiscovery/provider/vpc/command.py | 59 +-- .../provider/vpc/resource/analytics.py | 11 +- .../provider/vpc/resource/application.py | 5 +- .../provider/vpc/resource/compute.py | 20 +- .../provider/vpc/resource/containers.py | 6 +- .../provider/vpc/resource/database.py | 13 +- .../provider/vpc/resource/enduser.py | 6 +- .../provider/vpc/resource/identity.py | 5 +- .../provider/vpc/resource/management.py | 5 +- .../provider/vpc/resource/mediaservices.py | 12 +- cloudiscovery/provider/vpc/resource/ml.py | 16 +- .../provider/vpc/resource/network.py | 38 +- .../provider/vpc/resource/security.py | 8 +- .../provider/vpc/resource/storage.py | 10 +- cloudiscovery/shared/command.py | 158 +------- cloudiscovery/shared/common.py | 166 ++------ cloudiscovery/shared/common_aws.py | 347 +++++++++++++--- 30 files changed, 991 insertions(+), 837 deletions(-) diff --git a/README.md b/README.md index ca50598..c11117c 100644 --- a/README.md +++ b/README.md @@ -35,29 +35,29 @@ The commands generate reports that can be used to further analyze resources. 1.1 To detect AWS VPC resources (more on [AWS VPC](#aws-vpc)): ```sh -cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] +cloudiscovery aws-vpc [--vpc-id vpc-xxxxxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram [yes/no]] [--filter xxx] [--verbose] ``` 1.2 To detect AWS policy resources (more on [AWS Policy](#aws-policy)): ```sh -cloudiscovery aws-policy [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] +cloudiscovery aws-policy [--profile-name profile] [--diagram [yes/no]] [--filter xxx] [--verbose] ``` 1.3 To detect AWS IoT resources (more on [AWS IoT](#aws-iot)): ```sh -cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram True/False] [--filter xxx] [--verbose DEBUG] +cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--profile-name profile] [--diagram [yes/no]] [--filter xxx] [--verbose] ``` 1.4 To detect all AWS resources (more on [AWS All](#aws-all)): ```sh -cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose DEBUG] +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose] ``` 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): ```sh -cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--usage 0-100] [--verbose DEBUG] +cloudiscovery aws-limit --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--usage 0-100] [--verbose] ``` 2. For help use: @@ -147,7 +147,12 @@ The configured credentials must be associated to a user or role with proper perm "cloudhsm:DescribeClusters", "ssm:GetParametersByPath", "servicequotas:Get*", - "amplify:ListApps" + "amplify:ListApps", + "autoscaling-plans:DescribeScalingPlans", + "medialive:ListChannels", + "mediapackage:ListChannels", + "qldb:ListLedgers", + "transcribe:ListVocabularies" ], "Resource": [ "*" ] } @@ -338,7 +343,7 @@ cloudiscovery \ ``` -* If you are using Diagram output and due to fact container is a slim image of Python image, you must run cloudiscovery with "--diagram False", otherwise you'll have an error about "xdg-open". The output file will be saved in "assets/diagrams". +* If you are using Diagram output and due to fact container is a slim image of Python image, you must run cloudiscovery with "--diagram no", otherwise you'll have an error about "xdg-open". The output file will be saved in "assets/diagrams". ## Translate diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 413daee..3419926 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -35,11 +35,10 @@ from shared.common import ( exit_critical, - generate_session, Filterable, parse_filters, ) -from shared.common_aws import aws_verbose +from shared.common_aws import aws_verbose, generate_session # pylint: enable=wrong-import-position # Check version @@ -50,10 +49,21 @@ __version__ = "2.1.1" AVAILABLE_LANGUAGES = ["en_US", "pt_BR"] -DIAGRAMS_OPTIONS = ["True", "False"] DEFAULT_REGION = "us-east-1" +def str2bool(v): + if isinstance(v, bool): + return v + # pylint: disable=no-else-return + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("Boolean value expected.") + + def generate_parser(): parser = argparse.ArgumentParser() @@ -123,7 +133,13 @@ def add_default_arguments( "-l", "--language", required=False, help="Available languages: pt_BR, en_US" ) parser.add_argument( - "--verbose", "--verbose", required=False, help="Enable debug mode to sdk calls" + "--verbose", + "--verbose", + type=str2bool, + nargs="?", + const=True, + default=False, + help="Enable debug mode to sdk calls (default false)", ) if filters_enabled: parser.add_argument( @@ -139,9 +155,12 @@ def add_default_arguments( parser.add_argument( "-d", "--diagram", - required=False, - help='print diagram with resources (need Graphviz installed). Use options "True" to ' - 'view image or "False" to save image to disk. Default True', + type=str2bool, + nargs="?", + const=True, + default=True, + help="print diagram with resources (need Graphviz installed). Pass true/y[es] to " + "view image or false/n[0] not to generate image. Default true", ) @@ -158,7 +177,7 @@ def main(): # Check if verbose mode is enabled if args.verbose: - aws_verbose(verbose_mode=args.verbose) + aws_verbose() if args.language is None or args.language not in AVAILABLE_LANGUAGES: language = "en_US" @@ -167,9 +186,7 @@ def main(): # Diagram check if "diagram" not in args: - diagram = "False" - elif args.diagram is not None and args.diagram not in DIAGRAMS_OPTIONS: - diagram = "True" + diagram = False else: diagram = args.diagram @@ -184,8 +201,8 @@ def main(): check_diagram_version(diagram) # filters check + filters: List[Filterable] = [] if "filters" in args: - filters: List[Filterable] = [] if args.filters is not None: filters = parse_filters(args.filters) @@ -218,44 +235,29 @@ def main(): exit_critical(_("Threshold must be between 0 and 100")) if args.command == "aws-vpc": - command = Vpc( - vpc_id=args.vpc_id, - region_names=region_names, - session=session, - diagram=diagram, - filters=filters, - ) + command = Vpc(vpc_id=args.vpc_id, region_names=region_names, session=session,) elif args.command == "aws-policy": - command = Policy( - region_names=region_names, - session=session, - diagram=diagram, - filters=filters, - ) + command = Policy(region_names=region_names, session=session,) elif args.command == "aws-iot": command = Iot( - thing_name=args.thing_name, - region_names=region_names, - session=session, - diagram=diagram, - filters=filters, + thing_name=args.thing_name, region_names=region_names, session=session, ) elif args.command == "aws-all": - command = All(region_names=region_names, session=session, filters=filters,) + command = All(region_names=region_names, session=session) elif args.command == "aws-limit": command = Limit( - region_names=region_names, - session=session, - services=args.services, - threshold=args.threshold, + region_names=region_names, session=session, threshold=args.threshold, ) else: raise NotImplementedError("Unknown command") - command.run() + if "services" in args and args.services is not None: + services = args.services.split(",") + else: + services = [] + command.run(diagram, args.verbose, services, filters) def check_diagram_version(diagram): - if diagram: # Checking diagram version. Must be 0.13 or higher if pkg_resources.get_distribution("diagrams").version < "0.14": diff --git a/cloudiscovery/provider/all/command.py b/cloudiscovery/provider/all/command.py index fddf360..b819d74 100644 --- a/cloudiscovery/provider/all/command.py +++ b/cloudiscovery/provider/all/command.py @@ -1,25 +1,34 @@ -from shared.command import BaseCommand, CommandRunner -from shared.common import BaseAwsOptions +from typing import List + +from shared.common import Filterable, BaseOptions +from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram -class All(BaseCommand): - def __init__(self, region_names, session, filters): - """ - All AWS resources +class AllOptions(BaseAwsOptions, BaseOptions): + def __init__(self, verbose, filters, session, region_name): + BaseAwsOptions.__init__(self, session, region_name) + BaseOptions.__init__(self, verbose, filters) - :param region_names: - :param session: - :param filters: - """ - super().__init__(region_names, session, False, filters) - def run(self): +class All(BaseAwsCommand): + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): for region in self.region_names: self.init_region_cache(region) - options = BaseAwsOptions(session=self.session, region_name=region) + options = AllOptions( + verbose=verbose, + filters=filters, + session=self.session, + region_name=region, + ) - command_runner = CommandRunner(self.filters) + command_runner = AwsCommandRunner(filters) command_runner.run( provider="all", options=options, diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 474b0b0..ff1affe 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -6,16 +6,16 @@ from botocore.exceptions import UnknownServiceError from botocore.loaders import Loader +from provider.all.command import AllOptions from shared.common import ( ResourceProvider, Resource, - BaseAwsOptions, ResourceDigest, message_handler, ResourceAvailable, log_critical, - get_paginator, ) +from shared.common_aws import get_paginator OMITTED_RESOURCES = [ "aws_cloudhsm_available_zone", @@ -69,6 +69,7 @@ "aws_elasticbeanstalk_configuration_option", "aws_elasticbeanstalk_platform_version", "aws_iam_credential_report", + "aws_iam_account_password_policy", "aws_importexport_job", "aws_iot_o_taupdate", "aws_iot_default_authorizer", @@ -329,6 +330,8 @@ def wrapper(*args, **kwargs): # pylint: disable=broad-except except Exception as e: if func.__qualname__ == "AllResources.analyze_operation": + if not args[0].options.verbose: + return exception_str = str(e) if ( "is not subscribed to AWS Security Hub" in exception_str @@ -411,7 +414,7 @@ def build_resource_type(aws_service, name): class AllResources(ResourceProvider): - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: AllOptions): """ All resources @@ -459,15 +462,16 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): except UnknownServiceError: paginators_model = {"pagination": {}} service_full_name = service_model["metadata"]["serviceFullName"] - message_handler( - "Collecting data from {}...".format(service_full_name), "HEADER" - ) + if self.options.verbose: + message_handler( + "Collecting data from {}...".format(service_full_name), "HEADER" + ) if ( not self.availabilityCheck.is_service_available( self.options.region_name, aws_service ) or aws_service in SKIPPED_SERVICES - ): + ) and self.options.verbose: message_handler( "Service {} not available in this region... Skipping".format( service_full_name diff --git a/cloudiscovery/provider/iot/command.py b/cloudiscovery/provider/iot/command.py index 93467b6..fc31442 100644 --- a/cloudiscovery/provider/iot/command.py +++ b/cloudiscovery/provider/iot/command.py @@ -1,45 +1,45 @@ +from typing import List + from provider.iot.diagram import IoTDiagram -from shared.command import CommandRunner, BaseCommand -from shared.common import BaseAwsOptions, ResourceDigest +from shared.common import ResourceDigest, Filterable, BaseOptions +from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram, BaseDiagram -class IotOptions(BaseAwsOptions): +class IotOptions(BaseAwsOptions, BaseOptions): thing_name: str - def __new__(cls, session, region_name, thing_name): - """ - Iot options - - :param session: - :param region_name: - :param thing_name: - """ - self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) + # pylint: disable=too-many-arguments + def __init__(self, verbose, filters, session, region_name, thing_name): + BaseAwsOptions.__init__(self, session, region_name) + BaseOptions.__init__(self, verbose, filters) self.thing_name = thing_name - return self def iot_digest(self): return ResourceDigest(id=self.thing_name, type="aws_iot") -class Iot(BaseCommand): +class Iot(BaseAwsCommand): # pylint: disable=too-many-arguments - def __init__(self, thing_name, region_names, session, diagram, filters): + def __init__(self, thing_name, region_names, session): """ Iot command :param thing_name: :param region_names: :param session: - :param diagram: - :param filters: """ - super().__init__(region_names, session, diagram, filters) + super().__init__(region_names, session) self.thing_name = thing_name - def run(self): - command_runner = CommandRunner(self.filters) + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): + command_runner = AwsCommandRunner(filters) for region_name in self.region_names: self.init_region_cache(region_name) @@ -49,10 +49,14 @@ def run(self): client = self.session.client("iot", region_name=region_name) things = client.list_things() thing_options = IotOptions( - session=self.session, region_name=region_name, thing_name=things + verbose=verbose, + filters=filters, + session=self.session, + region_name=region_name, + thing_name=things, ) diagram_builder: BaseDiagram - if self.diagram: + if diagram: diagram_builder = IoTDiagram(thing_name="") else: diagram_builder = NoDiagram() @@ -67,10 +71,14 @@ def run(self): things = dict() things["things"] = [{"thingName": self.thing_name}] thing_options = IotOptions( - session=self.session, region_name=region_name, thing_name=things + verbose=verbose, + filters=filters, + session=self.session, + region_name=region_name, + thing_name=things, ) - if self.diagram: + if diagram: diagram_builder = IoTDiagram(thing_name=self.thing_name) else: diagram_builder = NoDiagram() diff --git a/cloudiscovery/provider/iot/resource/certificate.py b/cloudiscovery/provider/iot/resource/certificate.py index bb1bc50..d387b4f 100644 --- a/cloudiscovery/provider/iot/resource/certificate.py +++ b/cloudiscovery/provider/iot/resource/certificate.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -31,7 +31,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Certificates...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Certificates...", "HEADER") for thing in self.iot_options.thing_name["things"]: diff --git a/cloudiscovery/provider/iot/resource/policy.py b/cloudiscovery/provider/iot/resource/policy.py index 168a9d5..cbbe836 100644 --- a/cloudiscovery/provider/iot/resource/policy.py +++ b/cloudiscovery/provider/iot/resource/policy.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -31,7 +31,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Policies...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Policies...", "HEADER") for thing in self.iot_options.thing_name["things"]: diff --git a/cloudiscovery/provider/iot/resource/thing.py b/cloudiscovery/provider/iot/resource/thing.py index 6c7149b..9c67455 100644 --- a/cloudiscovery/provider/iot/resource/thing.py +++ b/cloudiscovery/provider/iot/resource/thing.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -30,7 +30,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Things...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Things...", "HEADER") for thing in self.iot_options.thing_name["things"]: client.describe_thing(thingName=thing["thingName"]) @@ -67,7 +68,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Things Type...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Things Type...", "HEADER") for thing in self.iot_options.thing_name["things"]: @@ -126,7 +128,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Jobs...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Jobs...", "HEADER") for thing in self.iot_options.thing_name["things"]: @@ -188,7 +191,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IoT Billing Group...", "HEADER") + if self.iot_options.verbose: + message_handler("Collecting data from IoT Billing Group...", "HEADER") for thing in self.iot_options.thing_name["things"]: diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index ecf9268..d26fc5b 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -1,59 +1,382 @@ from typing import List -from shared.command import BaseCommand, CommandRunner -from shared.common import BaseAwsOptions +from shared.common import ResourceCache, message_handler, Filterable, BaseOptions +from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram -from shared.common_aws import ALLOWED_SERVICES_CODES +ALLOWED_SERVICES_CODES = { + "acm": { + "L-F141DD1D": { + "method": "list_certificates", + "key": "CertificateSummaryList", + "fields": [], + }, + "global": False, + }, + "amplify": { + "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, + "global": False, + }, + "appmesh": { + "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, + "global": False, + }, + "appsync": { + "L-06A0647C": { + "method": "list_graphql_apis", + "key": "graphqlApis", + "fields": [], + }, + "global": False, + }, + "autoscaling-plans": { + "L-BD401546": { + "method": "describe_scaling_plans", + "key": "ScalingPlans", + "fields": [], + }, + "global": False, + }, + "batch": { + "L-144F0CA5": { + "method": "describe_compute_environments", + "key": "computeEnvironments", + "fields": [], + }, + "global": False, + }, + "codebuild": { + "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, + "global": False, + }, + "codecommit": { + "L-81790602": { + "method": "list_repositories", + "key": "repositories", + "fields": [], + }, + "global": False, + }, + "cloudformation": { + "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": [],}, + "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, + "global": False, + }, + "dynamodb": { + "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, + "global": False, + }, + "ec2": { + "L-0263D0A3": { + "method": "describe_addresses", + "key": "Addresses", + "fields": [], + }, + "global": False, + }, + "ecs": { + "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, + "global": False, + }, + "elasticfilesystem": { + "L-848C634D": { + "method": "describe_file_systems", + "key": "FileSystems", + "fields": [], + }, + "global": False, + }, + "elasticbeanstalk": { + "L-8EFC1C51": { + "method": "describe_environments", + "key": "Environments", + "fields": [], + }, + "L-1CEABD17": { + "method": "describe_applications", + "key": "Applications", + "fields": [], + }, + "global": False, + }, + "elasticloadbalancing": { + "L-53DA6B97": { + "method": "describe_load_balancers", + "key": "LoadBalancers", + "fields": [], + }, + "global": False, + }, + "iam": { + "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, + "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, + "L-BF35879D": { + "method": "list_server_certificates", + "key": "ServerCertificateMetadataList", + "fields": [], + }, + "L-6E65F664": { + "method": "list_instance_profiles", + "key": "InstanceProfiles", + "fields": [], + "paginate": False, + }, + "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, + "L-DB618D39": { + "method": "list_saml_providers", + "key": "SAMLProviderList", + "fields": [], + }, + "global": True, + }, + "kms": { + "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "global": False, + }, + "mediaconnect": { + "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, + "L-F1F62F5D": { + "method": "list_entitlements", + "key": "Entitlements", + "fields": [], + }, + "global": False, + }, + "medialive": { + "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, + "L-BDF24E14": { + "method": "list_input_devices", + "key": "InputDevices", + "fields": [], + }, + "global": False, + }, + "mediapackage": { + "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, + "global": False, + }, + "qldb": { + "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, + "global": False, + }, + "robomaker": { + "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, + "L-D6554FB1": { + "method": "list_simulation_applications", + "key": "simulationApplicationSummaries", + "fields": [], + }, + "global": False, + }, + "route53": { + "L-4EA4796A": { + "method": "list_hosted_zones", + "key": "HostedZones", + "fields": [], + }, + "L-ACB674F3": { + "method": "list_health_checks", + "key": "HealthChecks", + "fields": [], + }, + "global": True, + }, + "route53resolver": { + "L-4A669CC0": { + "method": "list_resolver_endpoints", + "key": "ResolverEndpoints", + "fields": [], + }, + "L-51D8A1FB": { + "method": "list_resolver_rules", + "key": "ResolverRules", + "fields": [], + }, + "global": True, + }, + "rds": { + "L-7B6409FD": { + "method": "describe_db_instances", + "key": "DBInstances", + "fields": [], + }, + "L-952B80B8": { + "method": "describe_db_clusters", + "key": "DBClusters", + "fields": [], + }, + "L-DE55804A": { + "method": "describe_db_parameter_groups", + "key": "DBParameterGroups", + "fields": [], + }, + "L-9FA33840": { + "method": "describe_option_groups", + "key": "OptionGroupsList", + "fields": [], + }, + "global": False, + }, + "s3": { + "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, + "global": False, + }, + "sns": { + "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, + "global": False, + }, + "transcribe": { + "L-3278D334": { + "method": "list_vocabularies", + "key": "Vocabularies", + "fields": [], + }, + "global": False, + }, + "translate": { + "L-4011ABD8": { + "method": "list_terminologies", + "key": "TerminologyPropertiesList", + "fields": [], + }, + "global": False, + }, +} -class LimitOptions(BaseAwsOptions): - services: List[str] - def __new__(cls, session, region_name, services, threshold): - """ - Limit Options +class LimitOptions(BaseAwsOptions, BaseOptions): + services: List[str] + threshold: str - :param session: - :param region_name: - :param services: - """ - self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) + # pylint: disable=too-many-arguments + def __init__( + self, + verbose: bool, + filters: List[Filterable], + session, + region_name, + services, + threshold, + ): + BaseAwsOptions.__init__(self, session, region_name) + BaseOptions.__init__(self, verbose, filters) self.services = services self.threshold = threshold - return self - -class Limit(BaseCommand): - def __init__(self, region_names, session, services, threshold): - """ - All AWS resources - :param region_names: - :param session: - :param services: - """ - super().__init__(region_names, session, False, False) +class LimitParameters: + def __init__(self, session, region: str, services): + self.region = region + self.cache = ResourceCache() + self.session = session self.services = [] if services is None: for service in ALLOWED_SERVICES_CODES: self.services.append(service) else: - self.services = services.split(",") + self.services = services + + def init_globalaws_limits_cache(self): + """ + AWS has global limit that can be adjustable and others that can't be adjustable + This method make cache for 15 days for aws cache global parameters. AWS don't update limit every time. + Services has differents limit, depending on region. + """ + for service_code in self.services: + if service_code in ALLOWED_SERVICES_CODES: + cache_key = "aws_limits_" + service_code + "_" + self.region + + cache = self.cache.get_key(cache_key) + if cache is not None: + continue + + message_handler( + "Fetching aws global limit to service {} in region {} to cache...".format( + service_code, self.region + ), + "HEADER", + ) + cache_codes = dict() + for quota_code in ALLOWED_SERVICES_CODES[service_code]: + + if quota_code != "global": + """ + Impossible to instance once at __init__ method. + Global services such route53 MUST USE us-east-1 region + """ + if ALLOWED_SERVICES_CODES[service_code]["global"]: + service_quota = self.session.client( + "service-quotas", region_name="us-east-1" + ) + else: + service_quota = self.session.client( + "service-quotas", region_name=self.region + ) + + response = service_quota.get_aws_default_service_quota( + ServiceCode=service_code, QuotaCode=quota_code + ) + + item_to_add = { + "value": response["Quota"]["Value"], + "adjustable": response["Quota"]["Adjustable"], + "quota_code": quota_code, + "quota_name": response["Quota"]["QuotaName"], + } + + if service_code in cache_codes: + cache_codes[service_code].append(item_to_add) + else: + cache_codes[service_code] = [item_to_add] + + self.cache.set_key(key=cache_key, value=cache_codes, expire=1296000) + + return True + + +class Limit(BaseAwsCommand): + def __init__(self, region_names, session, threshold): + """ + All AWS resources + + :param region_names: + :param session: + :param threshold: + """ + super().__init__(region_names, session) self.threshold = threshold - def run(self): + def init_globalaws_limits_cache(self, region, services): + # Cache services global and local services + LimitParameters( + session=self.session, region=region, services=services + ).init_globalaws_limits_cache() + + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): + if not services: + services = [] + for service in ALLOWED_SERVICES_CODES: + services.append(service) for region in self.region_names: - self.init_globalaws_limits_cache(region=region, services=self.services) + self.init_globalaws_limits_cache(region=region, services=services) limit_options = LimitOptions( + verbose=verbose, + filters=filters, session=self.session, region_name=region, - services=self.services, + services=services, threshold=self.threshold, ) - command_runner = CommandRunner(services=self.services) + command_runner = AwsCommandRunner(services=services) command_runner.run( provider="limit", options=limit_options, diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 3aee7a7..c53bae0 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -2,258 +2,18 @@ from concurrent.futures.thread import ThreadPoolExecutor -from shared.common import get_paginator - +from provider.limit.command import LimitOptions, ALLOWED_SERVICES_CODES from shared.common import ( ResourceProvider, Resource, - BaseAwsOptions, ResourceDigest, message_handler, ResourceCache, LimitsValues, ) +from shared.common_aws import get_paginator from shared.error_handler import exception -ALLOWED_SERVICES_CODES = { - "acm": { - "L-F141DD1D": { - "method": "list_certificates", - "key": "CertificateSummaryList", - "fields": [], - }, - "global": False, - }, - "amplify": { - "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, - "global": False, - }, - "appmesh": { - "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, - "global": False, - }, - "appsync": { - "L-06A0647C": { - "method": "list_graphql_apis", - "key": "graphqlApis", - "fields": [], - }, - "global": False, - }, - "autoscaling-plans": { - "L-BD401546": { - "method": "describe_scaling_plans", - "key": "ScalingPlans", - "fields": [], - }, - "global": False, - }, - "batch": { - "L-144F0CA5": { - "method": "describe_compute_environments", - "key": "computeEnvironments", - "fields": [], - }, - "global": False, - }, - "codebuild": { - "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, - "global": False, - }, - "codecommit": { - "L-81790602": { - "method": "list_repositories", - "key": "repositories", - "fields": [], - }, - "global": False, - }, - "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": [],}, - "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, - "global": False, - }, - "dynamodb": { - "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, - "global": False, - }, - "ec2": { - "L-0263D0A3": { - "method": "describe_addresses", - "key": "Addresses", - "fields": [], - }, - "global": False, - }, - "ecs": { - "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, - "global": False, - }, - "elasticfilesystem": { - "L-848C634D": { - "method": "describe_file_systems", - "key": "FileSystems", - "fields": [], - }, - "global": False, - }, - "elasticbeanstalk": { - "L-8EFC1C51": { - "method": "describe_environments", - "key": "Environments", - "fields": [], - }, - "L-1CEABD17": { - "method": "describe_applications", - "key": "Applications", - "fields": [], - }, - "global": False, - }, - "elasticloadbalancing": { - "L-53DA6B97": { - "method": "describe_load_balancers", - "key": "LoadBalancers", - "fields": [], - }, - "global": False, - }, - "iam": { - "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, - "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, - "L-BF35879D": { - "method": "list_server_certificates", - "key": "ServerCertificateMetadataList", - "fields": [], - }, - "L-6E65F664": { - "method": "list_instance_profiles", - "key": "InstanceProfiles", - "fields": [], - "paginate": False, - }, - "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, - "L-DB618D39": { - "method": "list_saml_providers", - "key": "SAMLProviderList", - "fields": [], - }, - "global": True, - }, - "kms": { - "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, - "global": False, - }, - "mediaconnect": { - "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, - "L-F1F62F5D": { - "method": "list_entitlements", - "key": "Entitlements", - "fields": [], - }, - "global": False, - }, - "medialive": { - "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, - "L-BDF24E14": { - "method": "list_input_devices", - "key": "InputDevices", - "fields": [], - }, - "global": False, - }, - "mediapackage": { - "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, - "global": False, - }, - "qldb": { - "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, - "global": False, - }, - "robomaker": { - "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, - "L-D6554FB1": { - "method": "list_simulation_applications", - "key": "simulationApplicationSummaries", - "fields": [], - }, - "global": False, - }, - "route53": { - "L-4EA4796A": { - "method": "list_hosted_zones", - "key": "HostedZones", - "fields": [], - }, - "L-ACB674F3": { - "method": "list_health_checks", - "key": "HealthChecks", - "fields": [], - }, - "global": True, - }, - "route53resolver": { - "L-4A669CC0": { - "method": "list_resolver_endpoints", - "key": "ResolverEndpoints", - "fields": [], - }, - "L-51D8A1FB": { - "method": "list_resolver_rules", - "key": "ResolverRules", - "fields": [], - }, - "global": True, - }, - "rds": { - "L-7B6409FD": { - "method": "describe_db_instances", - "key": "DBInstances", - "fields": [], - }, - "L-952B80B8": { - "method": "describe_db_clusters", - "key": "DBClusters", - "fields": [], - }, - "L-DE55804A": { - "method": "describe_db_parameter_groups", - "key": "DBParameterGroups", - "fields": [], - }, - "L-9FA33840": { - "method": "describe_option_groups", - "key": "OptionGroupsList", - "fields": [], - }, - "global": False, - }, - "s3": { - "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, - "global": False, - }, - "sns": { - "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, - "global": False, - }, - "transcribe": { - "L-3278D334": { - "method": "list_vocabularies", - "key": "Vocabularies", - "fields": [], - }, - "global": False, - }, - "translate": { - "L-4011ABD8": { - "method": "list_terminologies", - "key": "TerminologyPropertiesList", - "fields": [], - }, - "global": False, - }, -} - SERVICEQUOTA_TO_BOTO3 = { "elasticloadbalancing": "elbv2", "elasticfilesystem": "efs", @@ -263,7 +23,7 @@ class LimitResources(ResourceProvider): - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: LimitOptions): """ All resources @@ -343,14 +103,15 @@ def analyze_detail(self, client_quota, data_resource, service, threshold_request except client_quota.exceptions.NoSuchResourceException: value = data_quota_code["value"] - message_handler( - "Collecting data from Quota: " - + service - + " - " - + data_quota_code["quota_name"] - + "...", - "HEADER", - ) + if self.options.verbose: + message_handler( + "Collecting data from Quota: " + + service + + " - " + + data_quota_code["quota_name"] + + "...", + "HEADER", + ) # Need to convert some quota-services endpoint if service in SERVICEQUOTA_TO_BOTO3: diff --git a/cloudiscovery/provider/policy/command.py b/cloudiscovery/provider/policy/command.py index baf9884..561683f 100644 --- a/cloudiscovery/provider/policy/command.py +++ b/cloudiscovery/provider/policy/command.py @@ -1,17 +1,37 @@ +from typing import List + from provider.policy.diagram import PolicyDiagram -from shared.command import BaseCommand, CommandRunner -from shared.common import BaseAwsOptions + +from shared.common import Filterable, BaseOptions +from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram -class Policy(BaseCommand): - def run(self): +class PolicyOptions(BaseAwsOptions, BaseOptions): + def __init__(self, verbose, filters, session, region_name): + BaseAwsOptions.__init__(self, session, region_name) + BaseOptions.__init__(self, verbose, filters) + + +class Policy(BaseAwsCommand): + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): for region in self.region_names: self.init_region_cache(region) - options = BaseAwsOptions(session=self.session, region_name=region) + options = PolicyOptions( + verbose=verbose, + filters=filters, + session=self.session, + region_name=region, + ) - command_runner = CommandRunner(self.filters) - if self.diagram: + command_runner = AwsCommandRunner(filters) + if diagram: diagram = PolicyDiagram() else: diagram = NoDiagram() diff --git a/cloudiscovery/provider/policy/resource/general.py b/cloudiscovery/provider/policy/resource/general.py index f4b74de..5fbefaa 100644 --- a/cloudiscovery/provider/policy/resource/general.py +++ b/cloudiscovery/provider/policy/resource/general.py @@ -1,6 +1,6 @@ from typing import List -from shared.common import BaseAwsOptions, resource_tags +from provider.policy.command import PolicyOptions from shared.common import ( ResourceProvider, Resource, @@ -9,24 +9,27 @@ ResourceEdge, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception class IamUser(ResourceProvider): @ResourceAvailable(services="iam") - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: PolicyOptions): """ Iam user :param options: """ super().__init__() + self.options = options self.client = options.client("iam") self.users_found: List[Resource] = [] @exception def get_resources(self) -> List[Resource]: - message_handler("Collecting data from IAM Users...", "HEADER") + if self.options.verbose: + message_handler("Collecting data from IAM Users...", "HEADER") paginator = self.client.get_paginator("list_users") pages = paginator.paginate() diff --git a/cloudiscovery/provider/policy/resource/security.py b/cloudiscovery/provider/policy/resource/security.py index 6b1bcfe..4d18c11 100644 --- a/cloudiscovery/provider/policy/resource/security.py +++ b/cloudiscovery/provider/policy/resource/security.py @@ -1,7 +1,7 @@ from concurrent.futures.thread import ThreadPoolExecutor from typing import List -from shared.common import BaseAwsOptions, resource_tags +from provider.policy.command import PolicyOptions from shared.common import ( ResourceProvider, Resource, @@ -10,11 +10,11 @@ ResourceEdge, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception class Principals: - # Source: https://gist.github.com/shortjared/4c1e3fe52bdfa47522cfe5b41e5d6f22 principals = { "a4b.amazonaws.com": { @@ -812,7 +812,7 @@ class Principals: class IamPolicy(ResourceProvider): - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: PolicyOptions): """ Iam policy @@ -825,7 +825,8 @@ def __init__(self, options: BaseAwsOptions): @ResourceAvailable(services="iam") def get_resources(self) -> List[Resource]: client = self.options.client("iam") - message_handler("Collecting data from IAM Policies...", "HEADER") + if self.options.verbose: + message_handler("Collecting data from IAM Policies...", "HEADER") resources_found = [] @@ -855,20 +856,22 @@ def build_policy(data): class IamGroup(ResourceProvider): @ResourceAvailable(services="iam") - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: PolicyOptions): """ Iam group :param options: """ super().__init__() + self.options = options self.client = options.client("iam") self.resources_found: List[Resource] = [] @exception def get_resources(self) -> List[Resource]: - message_handler("Collecting data from IAM Groups...", "HEADER") + if self.options.verbose: + message_handler("Collecting data from IAM Groups...", "HEADER") paginator = self.client.get_paginator("list_groups") pages = paginator.paginate() @@ -917,20 +920,22 @@ def analyze_relations(self, resource): class IamRole(ResourceProvider): @ResourceAvailable(services="iam") - def __init__(self, options: BaseAwsOptions): + def __init__(self, options: PolicyOptions): """ Iam role :param options: """ super().__init__() + self.options = options self.client = options.client("iam") self.resources_found: List[Resource] = [] @exception def get_resources(self) -> List[Resource]: - message_handler("Collecting data from IAM Roles...", "HEADER") + if self.options.verbose: + message_handler("Collecting data from IAM Roles...", "HEADER") paginator = self.client.get_paginator("list_roles") pages = paginator.paginate() @@ -1031,7 +1036,7 @@ def analyze_role_relations(self, resource: Resource): class InstanceProfile(ResourceProvider): - def __init__(self, vpc_options: BaseAwsOptions): + def __init__(self, vpc_options: PolicyOptions): """ Instance profile @@ -1043,7 +1048,8 @@ def __init__(self, vpc_options: BaseAwsOptions): @exception def get_resources(self) -> List[Resource]: - message_handler("Collecting data from Instance Profiles...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Instance Profiles...", "HEADER") paginator = self.vpc_options.client("iam").get_paginator( "list_instance_profiles" ) diff --git a/cloudiscovery/provider/vpc/command.py b/cloudiscovery/provider/vpc/command.py index ccc1ffd..f8911d1 100644 --- a/cloudiscovery/provider/vpc/command.py +++ b/cloudiscovery/provider/vpc/command.py @@ -1,48 +1,45 @@ +from typing import List + from ipaddress import ip_network from provider.vpc.diagram import VpcDiagram -from shared.command import CommandRunner, BaseCommand from shared.common import ( - BaseAwsOptions, ResourceDigest, VPCE_REGEX, SOURCE_IP_ADDRESS_REGEX, + Filterable, + BaseOptions, ) +from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram, BaseDiagram -class VpcOptions(BaseAwsOptions): +class VpcOptions(BaseAwsOptions, BaseOptions): vpc_id: str - def __new__(cls, session, region_name, vpc_id): - """ - VPC Options - - :param session: - :param region_name: - :param vpc_id: - """ - self = super(BaseAwsOptions, cls).__new__(cls, (session, region_name)) + # pylint: disable=too-many-arguments + def __init__( + self, verbose: bool, filters: List[Filterable], session, region_name, vpc_id + ): + BaseAwsOptions.__init__(self, session, region_name) + BaseOptions.__init__(self, verbose, filters) self.vpc_id = vpc_id - return self def vpc_digest(self): return ResourceDigest(id=self.vpc_id, type="aws_vpc") -class Vpc(BaseCommand): +class Vpc(BaseAwsCommand): # pylint: disable=too-many-arguments - def __init__(self, vpc_id, region_names, session, diagram, filters): + def __init__(self, vpc_id, region_names, session): """ VPC command :param vpc_id: :param region_names: :param session: - :param diagram: - :param filters: """ - super().__init__(region_names, session, diagram, filters) + super().__init__(region_names, session) self.vpc_id = vpc_id @staticmethod @@ -64,9 +61,15 @@ def check_vpc(vpc_options: VpcOptions): ) print(message) - def run(self): + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): # pylint: disable=too-many-branches - command_runner = CommandRunner(self.filters) + command_runner = AwsCommandRunner(filters) for region in self.region_names: self.init_region_cache(region) @@ -78,11 +81,15 @@ def run(self): for data in vpcs["Vpcs"]: vpc_id = data["VpcId"] vpc_options = VpcOptions( - session=self.session, region_name=region, vpc_id=vpc_id, + verbose=verbose, + filters=filters, + session=self.session, + region_name=region, + vpc_id=vpc_id, ) self.check_vpc(vpc_options) diagram_builder: BaseDiagram - if self.diagram: + if diagram: diagram_builder = VpcDiagram(vpc_id=vpc_id) else: diagram_builder = NoDiagram() @@ -95,11 +102,15 @@ def run(self): ) else: vpc_options = VpcOptions( - session=self.session, region_name=region, vpc_id=self.vpc_id, + verbose=verbose, + filters=filters, + session=self.session, + region_name=region, + vpc_id=self.vpc_id, ) self.check_vpc(vpc_options) - if self.diagram: + if diagram: diagram_builder = VpcDiagram(vpc_id=self.vpc_id) else: diagram_builder = NoDiagram() diff --git a/cloudiscovery/provider/vpc/resource/analytics.py b/cloudiscovery/provider/vpc/resource/analytics.py index 9c225bc..89104fd 100644 --- a/cloudiscovery/provider/vpc/resource/analytics.py +++ b/cloudiscovery/provider/vpc/resource/analytics.py @@ -10,9 +10,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -36,7 +36,8 @@ def get_resources(self) -> List[Resource]: response = client.list_domain_names() - message_handler("Collecting data from Elasticsearch Domains...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Elasticsearch Domains...", "HEADER") for data in response["DomainNames"]: @@ -108,7 +109,8 @@ def get_resources(self) -> List[Resource]: # get all cache clusters response = client.list_clusters() - message_handler("Collecting data from MSK Clusters...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from MSK Clusters...", "HEADER") # iterate cache clusters to get subnet groups for data in response["ClusterInfoList"]: @@ -172,7 +174,8 @@ def get_resources(self) -> List[Resource]: response = client.list_data_sources(AwsAccountId=account_id) - message_handler("Collecting data from Quicksight...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Quicksight...", "HEADER") for data in response["DataSources"]: diff --git a/cloudiscovery/provider/vpc/resource/application.py b/cloudiscovery/provider/vpc/resource/application.py index c2a1770..55c9cf6 100644 --- a/cloudiscovery/provider/vpc/resource/application.py +++ b/cloudiscovery/provider/vpc/resource/application.py @@ -10,9 +10,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -36,7 +36,8 @@ def get_resources(self) -> List[Resource]: response = client.list_queues() - message_handler("Collecting data from SQS Queue Policy...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from SQS Queue Policy...", "HEADER") if "QueueUrls" in response: diff --git a/cloudiscovery/provider/vpc/resource/compute.py b/cloudiscovery/provider/vpc/resource/compute.py index c321b53..92e0450 100644 --- a/cloudiscovery/provider/vpc/resource/compute.py +++ b/cloudiscovery/provider/vpc/resource/compute.py @@ -7,12 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - get_name_tag, - get_tag, - resource_tags, ResourceAvailable, ) -from shared.common_aws import describe_subnet +from shared.common_aws import describe_subnet, resource_tags, get_name_tag, get_tag from shared.error_handler import exception @@ -31,7 +28,8 @@ def __init__(self, vpc_options: VpcOptions): def get_resources(self) -> List[Resource]: client = self.vpc_options.client("lambda") - message_handler("Collecting data from Lambda Functions...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Lambda Functions...", "HEADER") paginator = client.get_paginator("list_functions") pages = paginator.paginate() @@ -88,7 +86,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_instances() - message_handler("Collecting data from EC2 Instances...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from EC2 Instances...", "HEADER") for data in response["Reservations"]: for instances in data["Instances"]: @@ -155,7 +154,8 @@ def get_resources(self) -> List[Resource]: response = client.list_clusters() - message_handler("Collecting data from EKS Clusters...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from EKS Clusters...", "HEADER") for data in response["clusters"]: @@ -208,7 +208,8 @@ def get_resources(self) -> List[Resource]: response = client.list_clusters() - message_handler("Collecting data from EMR Clusters...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from EMR Clusters...", "HEADER") for data in response["Clusters"]: @@ -268,7 +269,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_auto_scaling_groups() - message_handler("Collecting data from Autoscaling Groups...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Autoscaling Groups...", "HEADER") for data in response["AutoScalingGroups"]: diff --git a/cloudiscovery/provider/vpc/resource/containers.py b/cloudiscovery/provider/vpc/resource/containers.py index 7983443..c261fa2 100644 --- a/cloudiscovery/provider/vpc/resource/containers.py +++ b/cloudiscovery/provider/vpc/resource/containers.py @@ -7,10 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) -from shared.common_aws import describe_subnet +from shared.common_aws import describe_subnet, resource_tags from shared.error_handler import exception @@ -39,7 +38,8 @@ def get_resources(self) -> List[Resource]: clusters=clusters_list["clusterArns"], include=["TAGS"] ) - message_handler("Collecting data from ECS Cluster...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from ECS Cluster...", "HEADER") # pylint: disable=too-many-nested-blocks for data in response["clusters"]: diff --git a/cloudiscovery/provider/vpc/resource/database.py b/cloudiscovery/provider/vpc/resource/database.py index afb5b82..2f59c7e 100644 --- a/cloudiscovery/provider/vpc/resource/database.py +++ b/cloudiscovery/provider/vpc/resource/database.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -56,7 +56,7 @@ def get_resources(self, instance_id=None) -> List[Resource]: response = client.describe_db_instances(Filters=[params]) - if instance_id is None: + if instance_id is None and self.vpc_options.verbose: message_handler("Collecting data from RDS Instances...", "HEADER") for data in response["DBInstances"]: @@ -116,7 +116,8 @@ def get_resources(self) -> List[Resource]: # get all cache clusters response = client.describe_cache_clusters() - message_handler("Collecting data from Elasticache Clusters...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Elasticache Clusters...", "HEADER") # iterate cache clusters to get subnet groups for data in response["CacheClusters"]: @@ -177,7 +178,8 @@ def get_resources(self) -> List[Resource]: Filters=[{"Name": "engine", "Values": ["docdb"]}] ) - message_handler("Collecting data from DocumentDB Instances...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from DocumentDB Instances...", "HEADER") # iterate cache clusters to get subnet groups for data in response["DBInstances"]: @@ -237,7 +239,8 @@ def get_resources(self) -> List[Resource]: Filters=[{"Name": "engine", "Values": ["neptune"]}] ) - message_handler("Collecting data from Neptune Instances...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Neptune Instances...", "HEADER") # iterate cache clusters to get subnet groups for data in response["DBInstances"]: diff --git a/cloudiscovery/provider/vpc/resource/enduser.py b/cloudiscovery/provider/vpc/resource/enduser.py index c9fdc04..f4dcd2d 100644 --- a/cloudiscovery/provider/vpc/resource/enduser.py +++ b/cloudiscovery/provider/vpc/resource/enduser.py @@ -7,10 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, - get_name_tag, ResourceAvailable, ) +from shared.common_aws import resource_tags, get_name_tag from shared.error_handler import exception @@ -34,7 +33,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_workspaces() - message_handler("Collecting data from Workspaces...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Workspaces...", "HEADER") for data in response["Workspaces"]: diff --git a/cloudiscovery/provider/vpc/resource/identity.py b/cloudiscovery/provider/vpc/resource/identity.py index a4a57c9..0d780bb 100644 --- a/cloudiscovery/provider/vpc/resource/identity.py +++ b/cloudiscovery/provider/vpc/resource/identity.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -33,7 +33,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_directories() - message_handler("Collecting data from Directory Services...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Directory Services...", "HEADER") for data in response["DirectoryDescriptions"]: diff --git a/cloudiscovery/provider/vpc/resource/management.py b/cloudiscovery/provider/vpc/resource/management.py index 3a23909..5c9cda9 100644 --- a/cloudiscovery/provider/vpc/resource/management.py +++ b/cloudiscovery/provider/vpc/resource/management.py @@ -7,9 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -33,7 +33,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_canaries() - message_handler("Collecting data from Synthetic Canaries...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Synthetic Canaries...", "HEADER") for data in response["Canaries"]: diff --git a/cloudiscovery/provider/vpc/resource/mediaservices.py b/cloudiscovery/provider/vpc/resource/mediaservices.py index ab2c47a..5254221 100644 --- a/cloudiscovery/provider/vpc/resource/mediaservices.py +++ b/cloudiscovery/provider/vpc/resource/mediaservices.py @@ -9,10 +9,9 @@ ResourceDigest, ResourceEdge, datetime_to_string, - resource_tags, ResourceAvailable, ) -from shared.common_aws import describe_subnet +from shared.common_aws import describe_subnet, resource_tags from shared.error_handler import exception @@ -36,7 +35,8 @@ def get_resources(self) -> List[Resource]: response = client.list_flows() - message_handler("Collecting data from Media Connect...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Media Connect...", "HEADER") for data in response["Flows"]: tags_response = client.list_tags_for_resource(ResourceArn=data["FlowArn"]) @@ -102,7 +102,8 @@ def get_resources(self) -> List[Resource]: response = client.list_inputs() - message_handler("Collecting data from Media Live Inputs...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Media Live Inputs...", "HEADER") for data in response["Inputs"]: tags_response = client.list_tags_for_resource(ResourceArn=data["Arn"]) @@ -154,7 +155,8 @@ def get_resources(self) -> List[Resource]: response = client.list_containers() - message_handler("Collecting data from Media Store...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Media Store...", "HEADER") for data in response["Containers"]: diff --git a/cloudiscovery/provider/vpc/resource/ml.py b/cloudiscovery/provider/vpc/resource/ml.py index b46a725..cdacb9e 100644 --- a/cloudiscovery/provider/vpc/resource/ml.py +++ b/cloudiscovery/provider/vpc/resource/ml.py @@ -7,10 +7,9 @@ message_handler, ResourceDigest, ResourceEdge, - resource_tags, ResourceAvailable, ) -from shared.common_aws import describe_subnet +from shared.common_aws import describe_subnet, resource_tags from shared.error_handler import exception @@ -34,9 +33,10 @@ def get_resources(self) -> List[Resource]: response = client.list_notebook_instances() - message_handler( - "Collecting data from Sagemaker Notebook instances...", "HEADER" - ) + if self.vpc_options.verbose: + message_handler( + "Collecting data from Sagemaker Notebook instances...", "HEADER" + ) for data in response["NotebookInstances"]: @@ -98,7 +98,8 @@ def get_resources(self) -> List[Resource]: response = client.list_training_jobs() - message_handler("Collecting data from Sagemaker Training Job...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Sagemaker Training Job...", "HEADER") for data in response["TrainingJobSummaries"]: tags_response = client.list_tags(ResourceArn=data["TrainingJobArn"],) @@ -165,7 +166,8 @@ def get_resources(self) -> List[Resource]: response = client.list_models() - message_handler("Collecting data from Sagemaker Model...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Sagemaker Model...", "HEADER") for data in response["Models"]: tags_response = client.list_tags(ResourceArn=data["ModelArn"],) diff --git a/cloudiscovery/provider/vpc/resource/network.py b/cloudiscovery/provider/vpc/resource/network.py index 24a49e2..7e6094c 100644 --- a/cloudiscovery/provider/vpc/resource/network.py +++ b/cloudiscovery/provider/vpc/resource/network.py @@ -7,13 +7,12 @@ ResourceProvider, Resource, message_handler, - get_name_tag, ResourceDigest, ResourceEdge, datetime_to_string, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags, get_name_tag from shared.error_handler import exception @@ -38,7 +37,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_internet_gateways(Filters=filters) - message_handler("Collecting data from Internet Gateways...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Internet Gateways...", "HEADER") # One VPC has only 1 IGW then it's a direct check if len(response["InternetGateways"]) > 0: @@ -94,7 +94,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_nat_gateways(Filters=filters) - message_handler("Collecting data from NAT Gateways...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from NAT Gateways...", "HEADER") for data in response["NatGateways"]: @@ -149,7 +150,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_load_balancers() - message_handler("Collecting data from Classic Load Balancers...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Classic Load Balancers...", "HEADER") for data in response["LoadBalancerDescriptions"]: if data["VPCId"] == self.vpc_options.vpc_id: @@ -199,7 +201,10 @@ def get_resources(self) -> List[Resource]: response = client.describe_load_balancers() - message_handler("Collecting data from Application Load Balancers...", "HEADER") + if self.vpc_options.verbose: + message_handler( + "Collecting data from Application Load Balancers...", "HEADER" + ) for data in response["LoadBalancers"]: @@ -256,7 +261,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_route_tables(Filters=filters) - message_handler("Collecting data from Route Tables...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Route Tables...", "HEADER") # Iterate to get all route table filtered for data in response["RouteTables"]: @@ -332,7 +338,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_subnets(Filters=filters) - message_handler("Collecting data from Subnets...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Subnets...", "HEADER") for data in response["Subnets"]: nametag = get_name_tag(data) @@ -383,7 +390,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_network_acls(Filters=filters) - message_handler("Collecting data from NACLs...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from NACLs...", "HEADER") for data in response["NetworkAcls"]: nacl_digest = ResourceDigest( @@ -438,7 +446,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_security_groups(Filters=filters) - message_handler("Collecting data from Security Groups...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from Security Groups...", "HEADER") for data in response["SecurityGroups"]: group_digest = ResourceDigest(id=data["GroupId"], type="aws_security_group") @@ -480,7 +489,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_vpc_peering_connections() - message_handler("Collecting data from VPC Peering...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from VPC Peering...", "HEADER") for data in response["VpcPeeringConnections"]: @@ -568,7 +578,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_vpc_endpoints(Filters=filters) - message_handler("Collecting data from VPC Endpoints...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from VPC Endpoints...", "HEADER") for data in response["VpcEndpoints"]: @@ -638,7 +649,8 @@ def get_resources(self) -> List[Resource]: # get REST API available response = client.get_rest_apis() - message_handler("Collecting data from REST API Policies...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from REST API Policies...", "HEADER") with ThreadPoolExecutor(15) as executor: results = executor.map(self.analyze_restapi, response["items"]) diff --git a/cloudiscovery/provider/vpc/resource/security.py b/cloudiscovery/provider/vpc/resource/security.py index a754394..0383bf1 100644 --- a/cloudiscovery/provider/vpc/resource/security.py +++ b/cloudiscovery/provider/vpc/resource/security.py @@ -10,9 +10,9 @@ ResourceDigest, ResourceEdge, datetime_to_string, - resource_tags, ResourceAvailable, ) +from shared.common_aws import resource_tags from shared.error_handler import exception @@ -34,7 +34,8 @@ def get_resources(self) -> List[Resource]: resources_found = [] - message_handler("Collecting data from IAM Policies...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from IAM Policies...", "HEADER") paginator = client.get_paginator("list_policies") pages = paginator.paginate(Scope="Local") for policies in pages: @@ -99,7 +100,8 @@ def get_resources(self) -> List[Resource]: response = client.describe_clusters() - message_handler("Collecting data from CloudHSM clusters...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from CloudHSM clusters...", "HEADER") for data in response["Clusters"]: diff --git a/cloudiscovery/provider/vpc/resource/storage.py b/cloudiscovery/provider/vpc/resource/storage.py index 4802ed9..1f14323 100644 --- a/cloudiscovery/provider/vpc/resource/storage.py +++ b/cloudiscovery/provider/vpc/resource/storage.py @@ -12,11 +12,9 @@ ResourceDigest, ResourceEdge, datetime_to_string, - resource_tags, - get_name_tag, ResourceAvailable, ) -from shared.common_aws import describe_subnet +from shared.common_aws import describe_subnet, resource_tags, get_name_tag from shared.error_handler import exception @@ -41,7 +39,8 @@ def get_resources(self) -> List[Resource]: # get filesystems available response = client.describe_file_systems() - message_handler("Collecting data from EFS Mount Targets...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from EFS Mount Targets...", "HEADER") for data in response["FileSystems"]: @@ -107,7 +106,8 @@ def get_resources(self) -> List[Resource]: # get buckets available response = client.list_buckets() - message_handler("Collecting data from S3 Bucket Policies...", "HEADER") + if self.vpc_options.verbose: + message_handler("Collecting data from S3 Bucket Policies...", "HEADER") with ThreadPoolExecutor(15) as executor: results = executor.map( diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index 18b629a..bd2c601 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -1,53 +1,12 @@ -import importlib -import inspect -import os -from concurrent.futures.thread import ThreadPoolExecutor -from os.path import dirname -from typing import Dict, List - -from boto3 import Session +from typing import List from shared.common import ( - ResourceProvider, Resource, - message_handler, - ResourceDigest, ResourceEdge, - BaseAwsOptions, Filterable, ResourceTag, ResourceType, ) -from shared.common_aws import GlobalParameters, LimitParameters -from shared.diagram import BaseDiagram -from shared.report import Report - - -class BaseCommand: - def __init__(self, region_names, session, diagram, filters): - """ - Base class for discovery command - - :param region_names: - :param session: - :param diagram: - :param filters: - """ - self.region_names: List[str] = region_names - self.session: Session = session - self.diagram: bool = diagram - self.filters: List[Filterable] = filters - - def init_region_cache(self, region): - # Get and cache SSM services available in specific region - path = "/aws/service/global-infrastructure/regions/" + region + "/services/" - GlobalParameters(session=self.session, region=region, path=path).paths() - - def init_globalaws_limits_cache(self, region, services): - # Cache services global and local services - LimitParameters( - session=self.session, region=region, services=services - ).init_globalaws_limits_cache() def filter_resources( @@ -98,118 +57,3 @@ def execute_provider(options, data) -> (List[Resource], List[ResourceEdge]): provider_resources = provider_instance.get_resources() provider_resource_relations = provider_instance.get_relations() return provider_resources, provider_resource_relations - - -class CommandRunner(object): - def __init__(self, filters=None, services=None): - """ - Base class command execution - - :param filters: - """ - self.filters: List[Filterable] = filters - self.services: List[str] = services - - # pylint: disable=too-many-locals,too-many-arguments - def run( - self, - provider: str, - options: BaseAwsOptions, - diagram_builder: BaseDiagram, - title: str, - filename: str, - ): - """ - Executes a command. - - The project's development pattern is a file with the respective name of the parent - resource (e.g. compute, network), classes of child resources inside this file and run() method to execute - respective check. So it makes sense to load dynamically. - """ - # Iterate to get all modules - message_handler("\nInspecting resources", "HEADER") - providers = [] - for name in os.listdir( - dirname(__file__) + "/../provider/" + provider + "/resource" - ): - if name.endswith(".py"): - # strip the extension - module = name[:-3] - - # Load and call all run check - for nameclass, cls in inspect.getmembers( - importlib.import_module( - "provider." + provider + ".resource." + module - ), - inspect.isclass, - ): - if ( - issubclass(cls, ResourceProvider) - and cls is not ResourceProvider - ): - providers.append((nameclass, cls)) - providers.sort(key=lambda x: x[0]) - - all_resources: List[Resource] = [] - resource_relations: List[ResourceEdge] = [] - - with ThreadPoolExecutor(15) as executor: - provider_results = executor.map( - lambda data: execute_provider(options, data), providers - ) - - for provider_results in provider_results: - if provider_results[0] is not None: - all_resources.extend(provider_results[0]) - if provider_results[1] is not None: - resource_relations.extend(provider_results[1]) - - unique_resources_dict: Dict[ResourceDigest, Resource] = dict() - for resource in all_resources: - unique_resources_dict[resource.digest] = resource - - unique_resources = list(unique_resources_dict.values()) - - unique_resources.sort(key=lambda x: x.group + x.digest.type + x.name) - resource_relations.sort( - key=lambda x: x.from_node.type - + x.from_node.id - + x.to_node.type - + x.to_node.id - ) - - # Resource filtering and sorting - filtered_resources = filter_resources(unique_resources, self.filters) - filtered_resources.sort(key=lambda x: x.group + x.digest.type + x.name) - - # Relationships filtering and sorting - filtered_relations = filter_relations(filtered_resources, resource_relations) - filtered_relations.sort( - key=lambda x: x.from_node.type - + x.from_node.id - + x.to_node.type - + x.to_node.id - ) - - # Diagram integration - diagram_builder.build( - resources=filtered_resources, - resource_relations=filtered_relations, - title=title, - filename=filename, - ) - - # TODO: Generate reports in json/csv/pdf/xls - report = Report() - report.general_report( - resources=filtered_resources, resource_relations=filtered_relations - ), - report.html_report( - resources=filtered_resources, - resource_relations=filtered_relations, - title=title, - filename=filename, - ) - - # TODO: Export in csv/json/yaml/tf... future... - # ....exporttf(checks).... diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 8e54a27..0b0332d 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -3,12 +3,11 @@ import re import functools import threading -from typing import NamedTuple, List, Optional, Dict +from abc import ABC +from typing import NamedTuple, List from diskcache import Cache -import boto3 - VPCE_REGEX = re.compile(r'(?<=sourcevpce")(\s*:\s*")(vpce-[a-zA-Z0-9]+)', re.DOTALL) SOURCE_IP_ADDRESS_REGEX = re.compile( r'(?<=sourceip")(\s*:\s*")([a-fA-F0-9.:/%]+)', re.DOTALL @@ -34,22 +33,6 @@ class bcolors: } -class BaseAwsOptions(NamedTuple): - session: boto3.Session - region_name: str - - def client(self, service_name: str): - return self.session.client(service_name, region_name=self.region_name) - - def resulting_file_name(self, suffix): - return "{}_{}_{}".format(self.account_number(), self.region_name, suffix) - - def account_number(self): - client = self.session.client("sts", region_name=self.region_name) - account_id = client.get_caller_identity()["Account"] - return account_id - - class ResourceDigest(NamedTuple): id: str type: str @@ -135,80 +118,27 @@ def wrapper(*args, **kwargs): if self.is_service_available(region_name, self.services): return func(*args, **kwargs) - message_handler( - "Check " - + func.__qualname__ - + " not available in this region... Skipping", - "WARNING", - ) + verbose = False + if "vpc_options" in dir(args[0]): + verbose = args[0].vpc_options.verbose + elif "iot_options" in dir(args[0]): + verbose = args[0].iot_options.verbose + elif "options" in dir(args[0]): + verbose = args[0].options.verbose + + if verbose: + message_handler( + "Check " + + func.__qualname__ + + " not available in this region... Skipping", + "WARNING", + ) return None return wrapper -def resource_tags(resource_data: dict) -> List[ResourceTag]: - if "Tags" in resource_data: - tags_input = resource_data["Tags"] - elif "tags" in resource_data: - tags_input = resource_data["tags"] - elif "TagList" in resource_data: - tags_input = resource_data["TagList"] - elif "TagSet" in resource_data: - tags_input = resource_data["TagSet"] - else: - tags_input = None - - tags = [] - if isinstance(tags_input, list): - tags = resource_tags_from_tuples(tags_input) - elif isinstance(tags_input, dict): - tags = resource_tags_from_dict(tags_input) - - return tags - - -def resource_tags_from_tuples(tuples: List[Dict[str, str]]) -> List[ResourceTag]: - """ - List of key-value tuples that store tags, syntax: - [ - { - 'Key': 'string', - 'Value': 'string', - ... - }, - ] - OR - [ - { - 'key': 'string', - 'value': 'string', - ... - }, - ] - """ - result = [] - for tuple_elem in tuples: - if "Key" in tuple_elem and "Value" in tuple_elem: - result.append(ResourceTag(key=tuple_elem["Key"], value=tuple_elem["Value"])) - elif "key" in tuple_elem and "value" in tuple_elem: - result.append(ResourceTag(key=tuple_elem["key"], value=tuple_elem["value"])) - return result - - -def resource_tags_from_dict(tags: Dict[str, str]) -> List[ResourceTag]: - """ - List of key-value dict that store tags, syntax: - { - 'string': 'string' - } - """ - result = [] - for key, value in tags.items(): - result.append(ResourceTag(key=key, value=value)) - return result - - class ResourceProvider: def __init__(self): """ @@ -225,31 +155,6 @@ def get_relations(self) -> List[ResourceEdge]: return self.relations_found -def get_name_tag(d) -> Optional[str]: - return get_tag(d, "Name") - - -def get_tag(d, tag_name) -> Optional[str]: - for k, v in d.items(): - if k in ("Tags", "TagList"): - for value in v: - if value["Key"] == tag_name: - return value["Value"] - - return None - - -def generate_session(profile_name): - try: - return boto3.Session(profile_name=profile_name) - # pylint: disable=broad-except - except Exception as e: - message = "You must configure awscli before use this script.\nError: {0}".format( - str(e) - ) - exit_critical(message) - - def exit_critical(message): log_critical(message) raise SystemExit @@ -322,20 +227,25 @@ def parse_filters(arg_filters) -> List[Filterable]: return filters -def get_paginator(client, operation_name, resource_type): - """ - TODO: Possible circular reference using in common_aws, move to there in future. - """ - # Checking if can paginate - if client.can_paginate(operation_name): - paginator = client.get_paginator(operation_name) - if resource_type == "aws_iam_policy": - pages = paginator.paginate( - Scope="Local" - ) # hack to list only local IAM policies - aws_all - else: - pages = paginator.paginate() - else: - return False +class BaseCommand(ABC): + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): + raise NotImplementedError() + + +class Object(object): + pass + + +class BaseOptions(Object): + verbose: bool + filters: List[Filterable] - return pages + def __init__(self, verbose: bool, filters: List[Filterable]): + self.verbose = verbose + self.filters = filters diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index cb6cbbe..70ebe9b 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -1,9 +1,31 @@ +import importlib +import inspect +import os +from abc import ABC +from concurrent.futures.thread import ThreadPoolExecutor +from os.path import dirname +from typing import List, Dict, Optional + import botocore.exceptions import boto3 +from boto3 import Session from cachetools import TTLCache -from shared.common import ResourceCache, message_handler -from provider.limit.resource.all import ALLOWED_SERVICES_CODES +from shared.command import execute_provider, filter_resources, filter_relations +from shared.common import ( + ResourceCache, + message_handler, + ResourceTag, + ResourceProvider, + Resource, + ResourceEdge, + ResourceDigest, + Filterable, + exit_critical, + BaseCommand, +) +from shared.diagram import BaseDiagram +from shared.report import Report SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) @@ -24,7 +46,7 @@ def describe_subnet(vpc_options, subnet_ids): return None -def aws_verbose(verbose_mode="DEBUG"): +def aws_verbose(): """ Boto3 only provides usable information in DEBUG mode Using empty name it catchs debug from boto3/botocore @@ -33,75 +55,30 @@ def aws_verbose(verbose_mode="DEBUG"): boto3.set_stream_logger(name="") -class LimitParameters: - def __init__(self, session, region: str, services): - self.region = region - self.cache = ResourceCache() - self.session = session - self.services = [] - if services is None: - for service in ALLOWED_SERVICES_CODES: - self.services.append(service) - else: - self.services = services +class BaseAwsOptions: + session: boto3.Session + region_name: str - def init_globalaws_limits_cache(self): + def __init__(self, session, region_name): """ - AWS has global limit that can be adjustable and others that can't be adjustable - This method make cache for 15 days for aws cache global parameters. AWS don't update limit every time. - Services has differents limit, depending on region. + Base AWS options + + :param session: + :param region_name: """ - for service_code in self.services: - if service_code in ALLOWED_SERVICES_CODES: - cache_key = "aws_limits_" + service_code + "_" + self.region + self.session = session + self.region_name = region_name - cache = self.cache.get_key(cache_key) - if cache is not None: - continue + def client(self, service_name: str): + return self.session.client(service_name, region_name=self.region_name) - message_handler( - "Fetching aws global limit to service {} in region {} to cache...".format( - service_code, self.region - ), - "HEADER", - ) - - cache_codes = dict() - for quota_code in ALLOWED_SERVICES_CODES[service_code]: - - if quota_code != "global": - """ - Impossible to instance once at __init__ method. - Global services such route53 MUST USE us-east-1 region - """ - if ALLOWED_SERVICES_CODES[service_code]["global"]: - service_quota = self.session.client( - "service-quotas", region_name="us-east-1" - ) - else: - service_quota = self.session.client( - "service-quotas", region_name=self.region - ) - - response = service_quota.get_aws_default_service_quota( - ServiceCode=service_code, QuotaCode=quota_code - ) - - item_to_add = { - "value": response["Quota"]["Value"], - "adjustable": response["Quota"]["Adjustable"], - "quota_code": quota_code, - "quota_name": response["Quota"]["QuotaName"], - } - - if service_code in cache_codes: - cache_codes[service_code].append(item_to_add) - else: - cache_codes[service_code] = [item_to_add] - - self.cache.set_key(key=cache_key, value=cache_codes, expire=1296000) - - return True + def resulting_file_name(self, suffix): + return "{}_{}_{}".format(self.account_number(), self.region_name, suffix) + + def account_number(self): + client = self.session.client("sts", region_name=self.region_name) + account_id = client.get_caller_identity()["Account"] + return account_id class GlobalParameters: @@ -151,3 +128,239 @@ def paths(self): self.cache.set_key(key=cache_key, value=paths_found, expire=86400) return paths_found + + +class BaseAwsCommand(BaseCommand, ABC): + def __init__(self, region_names, session): + """ + Base class for discovery command + + :param region_names: + :param session: + """ + self.region_names: List[str] = region_names + self.session: Session = session + + def init_region_cache(self, region): + # Get and cache SSM services available in specific region + path = "/aws/service/global-infrastructure/regions/" + region + "/services/" + GlobalParameters(session=self.session, region=region, path=path).paths() + + +def resource_tags(resource_data: dict) -> List[ResourceTag]: + if "Tags" in resource_data: + tags_input = resource_data["Tags"] + elif "tags" in resource_data: + tags_input = resource_data["tags"] + elif "TagList" in resource_data: + tags_input = resource_data["TagList"] + elif "TagSet" in resource_data: + tags_input = resource_data["TagSet"] + else: + tags_input = None + + tags = [] + if isinstance(tags_input, list): + tags = resource_tags_from_tuples(tags_input) + elif isinstance(tags_input, dict): + tags = resource_tags_from_dict(tags_input) + + return tags + + +def resource_tags_from_tuples(tuples: List[Dict[str, str]]) -> List[ResourceTag]: + """ + List of key-value tuples that store tags, syntax: + [ + { + 'Key': 'string', + 'Value': 'string', + ... + }, + ] + OR + [ + { + 'key': 'string', + 'value': 'string', + ... + }, + ] + """ + result = [] + for tuple_elem in tuples: + if "Key" in tuple_elem and "Value" in tuple_elem: + result.append(ResourceTag(key=tuple_elem["Key"], value=tuple_elem["Value"])) + elif "key" in tuple_elem and "value" in tuple_elem: + result.append(ResourceTag(key=tuple_elem["key"], value=tuple_elem["value"])) + return result + + +def resource_tags_from_dict(tags: Dict[str, str]) -> List[ResourceTag]: + """ + List of key-value dict that store tags, syntax: + { + 'string': 'string' + } + """ + result = [] + for key, value in tags.items(): + result.append(ResourceTag(key=key, value=value)) + return result + + +def get_name_tag(d) -> Optional[str]: + return get_tag(d, "Name") + + +def get_tag(d, tag_name) -> Optional[str]: + for k, v in d.items(): + if k in ("Tags", "TagList"): + for value in v: + if value["Key"] == tag_name: + return value["Value"] + + return None + + +def generate_session(profile_name): + try: + return boto3.Session(profile_name=profile_name) + # pylint: disable=broad-except + except Exception as e: + message = "You must configure awscli before use this script.\nError: {0}".format( + str(e) + ) + exit_critical(message) + + +def get_paginator(client, operation_name, resource_type): + # Checking if can paginate + if client.can_paginate(operation_name): + paginator = client.get_paginator(operation_name) + if resource_type == "aws_iam_policy": + pages = paginator.paginate( + Scope="Local" + ) # hack to list only local IAM policies - aws_all + else: + pages = paginator.paginate() + else: + return False + + return pages + + +class AwsCommandRunner(object): + def __init__(self, services=None, filters=None): + """ + Base class command execution + + :param services: + :param filters: + """ + self.services: List[str] = services + self.filters: List[Filterable] = filters + + # pylint: disable=too-many-locals,too-many-arguments + def run( + self, + provider: str, + options: BaseAwsOptions, + diagram_builder: BaseDiagram, + title: str, + filename: str, + ): + """ + Executes a command. + + The project's development pattern is a file with the respective name of the parent + resource (e.g. compute, network), classes of child resources inside this file and run() method to execute + respective check. So it makes sense to load dynamically. + """ + # Iterate to get all modules + message_handler("\nInspecting resources", "HEADER") + providers = [] + for name in os.listdir( + dirname(__file__) + "/../provider/" + provider + "/resource" + ): + if name.endswith(".py"): + # strip the extension + module = name[:-3] + + # Load and call all run check + for nameclass, cls in inspect.getmembers( + importlib.import_module( + "provider." + provider + ".resource." + module + ), + inspect.isclass, + ): + if ( + issubclass(cls, ResourceProvider) + and cls is not ResourceProvider + ): + providers.append((nameclass, cls)) + providers.sort(key=lambda x: x[0]) + + all_resources: List[Resource] = [] + resource_relations: List[ResourceEdge] = [] + + with ThreadPoolExecutor(15) as executor: + provider_results = executor.map( + lambda data: execute_provider(options, data), providers + ) + + for provider_results in provider_results: + if provider_results[0] is not None: + all_resources.extend(provider_results[0]) + if provider_results[1] is not None: + resource_relations.extend(provider_results[1]) + + unique_resources_dict: Dict[ResourceDigest, Resource] = dict() + for resource in all_resources: + unique_resources_dict[resource.digest] = resource + + unique_resources = list(unique_resources_dict.values()) + + unique_resources.sort(key=lambda x: x.group + x.digest.type + x.name) + resource_relations.sort( + key=lambda x: x.from_node.type + + x.from_node.id + + x.to_node.type + + x.to_node.id + ) + + # Resource filtering and sorting + filtered_resources = filter_resources(unique_resources, self.filters) + filtered_resources.sort(key=lambda x: x.group + x.digest.type + x.name) + + # Relationships filtering and sorting + filtered_relations = filter_relations(filtered_resources, resource_relations) + filtered_relations.sort( + key=lambda x: x.from_node.type + + x.from_node.id + + x.to_node.type + + x.to_node.id + ) + + # Diagram integration + diagram_builder.build( + resources=filtered_resources, + resource_relations=filtered_relations, + title=title, + filename=filename, + ) + + # TODO: Generate reports in json/csv/pdf/xls + report = Report() + report.general_report( + resources=filtered_resources, resource_relations=filtered_relations + ), + report.html_report( + resources=filtered_resources, + resource_relations=filtered_relations, + title=title, + filename=filename, + ) + + # TODO: Export in csv/json/yaml/tf... future... + # ....exporttf(checks).... From 77ca9adcd1967c31e96407216252f59f1ebf4e3d Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 01:22:11 +0700 Subject: [PATCH 74/86] pylint codacy issue --- cloudiscovery/shared/common_aws.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index 70ebe9b..f4aa2a3 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -1,7 +1,6 @@ import importlib import inspect import os -from abc import ABC from concurrent.futures.thread import ThreadPoolExecutor from os.path import dirname from typing import List, Dict, Optional @@ -130,7 +129,7 @@ def paths(self): return paths_found -class BaseAwsCommand(BaseCommand, ABC): +class BaseAwsCommand(BaseCommand): def __init__(self, region_names, session): """ Base class for discovery command @@ -141,6 +140,15 @@ def __init__(self, region_names, session): self.region_names: List[str] = region_names self.session: Session = session + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + ): + raise NotImplementedError() + def init_region_cache(self, region): # Get and cache SSM services available in specific region path = "/aws/service/global-infrastructure/regions/" + region + "/services/" From d58441a85aef3f0490782a755fa9db47f5277f82 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 12:58:40 +0700 Subject: [PATCH 75/86] adjusted help text --- cloudiscovery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 3419926..372a8e5 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -160,7 +160,7 @@ def add_default_arguments( const=True, default=True, help="print diagram with resources (need Graphviz installed). Pass true/y[es] to " - "view image or false/n[0] not to generate image. Default true", + "view image or false/n[o] not to generate image. Default true", ) From 822ddb1d9b79e2a266169393c59c4d9223993a86 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 14:42:48 +0700 Subject: [PATCH 76/86] added service narrowing for all command, improved its reporting --- README.md | 8 +- cloudiscovery/__init__.py | 19 +++-- cloudiscovery/provider/all/command.py | 7 +- cloudiscovery/provider/all/resource/all.py | 97 ++++++++++++++++------ cloudiscovery/shared/common.py | 5 +- cloudiscovery/shared/report.py | 54 ++++++++---- cloudiscovery/templates/report_html.html | 9 +- 7 files changed, 145 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index c11117c..26ac53e 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--pro 1.4 To detect all AWS resources (more on [AWS All](#aws-all)): ```sh -cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose] +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--filter xxx] [--verbose] ``` 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): @@ -272,13 +272,13 @@ Following resources are checked in IoT command: ### AWS All -List all AWS resources (preview) +A command to list **ALL** AWS resources. -The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). +The command calls all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). -Types of resources mostly cover Terraform types. +Types of resources mostly cover Terraform types. It is possible to narrow down scope of the resources to ones related with a given service with parameter `-s` e.g. `-s ec2,ecs,cloudfront,rds`. ### AWS Limit diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 372a8e5..b9ac67b 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -92,18 +92,13 @@ def generate_parser(): all_parser = subparsers.add_parser("aws-all", help="Analyze all resources") add_default_arguments(all_parser, diagram_enabled=False) + add_services_argument(all_parser) limit_parser = subparsers.add_parser( "aws-limit", help="Analyze aws limit resources." ) add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False) - limit_parser.add_argument( - "-s", - "--services", - required=False, - help='Inform services that you want to check, use "," (comma) to split them. \ - If not informed, script will check all services.', - ) + add_services_argument(limit_parser) limit_parser.add_argument( "-t", "--threshold", @@ -115,6 +110,16 @@ def generate_parser(): return parser +def add_services_argument(limit_parser): + limit_parser.add_argument( + "-s", + "--services", + required=False, + help='Define services that you want to check, use "," (comma) to separate multiple names. \ + If not passed, command will check all services.', + ) + + def add_default_arguments( parser, is_global=False, diagram_enabled=True, filters_enabled=True ): diff --git a/cloudiscovery/provider/all/command.py b/cloudiscovery/provider/all/command.py index b819d74..72c0f25 100644 --- a/cloudiscovery/provider/all/command.py +++ b/cloudiscovery/provider/all/command.py @@ -6,9 +6,13 @@ class AllOptions(BaseAwsOptions, BaseOptions): - def __init__(self, verbose, filters, session, region_name): + services: List[str] + + # pylint: disable=too-many-arguments + def __init__(self, verbose, filters, session, region_name, services: List[str]): BaseAwsOptions.__init__(self, session, region_name) BaseOptions.__init__(self, verbose, filters) + self.services = services class All(BaseAwsCommand): @@ -26,6 +30,7 @@ def run( filters=filters, session=self.session, region_name=region, + services=services, ) command_runner = AwsCommandRunner(filters) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index ff1affe..070921b 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -1,3 +1,4 @@ +import collections import functools import re from concurrent.futures.thread import ThreadPoolExecutor @@ -15,7 +16,7 @@ ResourceAvailable, log_critical, ) -from shared.common_aws import get_paginator +from shared.common_aws import get_paginator, resource_tags OMITTED_RESOURCES = [ "aws_cloudhsm_available_zone", @@ -309,7 +310,9 @@ def operation_allowed( return evaluation_result -def build_resource(base_resource, operation_name, resource_type) -> Optional[Resource]: +def build_resource( + base_resource, operation_name, resource_type, group +) -> Optional[Resource]: if isinstance(base_resource, str): return None resource_name = retrieve_resource_name(base_resource, operation_name) @@ -317,8 +320,13 @@ def build_resource(base_resource, operation_name, resource_type) -> Optional[Res if resource_id is None or resource_name is None: return None + attributes = flatten(base_resource) return Resource( - digest=ResourceDigest(id=resource_id, type=resource_type), name=resource_name, + digest=ResourceDigest(id=resource_id, type=resource_type), + group=group, + name=resource_name, + attributes=attributes, + tags=resource_tags(base_resource), ) @@ -413,6 +421,17 @@ def build_resource_type(aws_service, name): ) +def flatten(d, parent_key="", sep="."): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + class AllResources(ResourceProvider): def __init__(self, options: AllOptions): """ @@ -427,16 +446,20 @@ def __init__(self, options: AllOptions): @all_exception def get_resources(self) -> List[Resource]: boto_loader = Loader() - aws_services = boto_loader.list_available_services(type_name="service-2") + if self.options.services: + aws_services = self.options.services + else: + aws_services = boto_loader.list_available_services(type_name="service-2") resources = [] allowed_actions = self.get_policies_allowed_actions() - message_handler( - "Analyzing listing operations across {} service...".format( - len(aws_services) - ), - "HEADER", - ) + if self.options.verbose: + message_handler( + "Analyzing listing operations across {} service...".format( + len(aws_services) + ), + "HEADER", + ) with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor: results = executor.map( lambda aws_service: self.analyze_service( @@ -501,7 +524,12 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): if not operation_allowed(allowed_actions, aws_service, name): continue analyze_operation = self.analyze_operation( - resource_type, name, has_paginator, client, service_full_name + resource_type, + name, + has_paginator, + client, + service_full_name, + aws_service, ) if analyze_operation is not None: resources.extend(analyze_operation) @@ -510,10 +538,17 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): @all_exception # pylint: disable=too-many-locals,too-many-arguments def analyze_operation( - self, resource_type, operation_name, has_paginator, client, service_full_name + self, + resource_type, + operation_name, + has_paginator, + client, + service_full_name, + aws_service, ) -> List[Resource]: resources = [] snake_operation_name = _to_snake_case(operation_name) + # pylint: disable=too-many-nested-blocks if has_paginator: pages = get_paginator( client=client, @@ -530,14 +565,23 @@ def analyze_operation( result_parent = list_metadata["children"][0]["value"] result_child = list_metadata["children"][1]["value"] else: - message_handler( - "Operation {} has unsupported pagination definition... Skipping".format( - snake_operation_name - ), - "WARNING", - ) + if self.options.verbose: + message_handler( + "Operation {} has unsupported pagination definition... Skipping".format( + snake_operation_name + ), + "WARNING", + ) return [] for page in pages: + if result_key == "Reservations": # hack for EC2 instances + for page_reservation in page["Reservations"]: + for instance in page_reservation["Instances"]: + resource = build_resource( + instance, operation_name, resource_type, aws_service + ) + if resource is not None: + resources.append(resource) if result_key is not None: page_resources = page[result_key] elif result_child in page[result_parent]: @@ -546,7 +590,7 @@ def analyze_operation( page_resources = [] for page_resource in page_resources: resource = build_resource( - page_resource, operation_name, resource_type + page_resource, operation_name, resource_type, aws_service ) if resource is not None: resources.append(resource) @@ -557,14 +601,18 @@ def analyze_operation( if isinstance(response_elem, list): for response_resource in response_elem: resource = build_resource( - response_resource, operation_name, resource_type + response_resource, + operation_name, + resource_type, + aws_service, ) if resource is not None: resources.append(resource) return resources def get_policies_allowed_actions(self): - message_handler("Fetching allowed actions...", "HEADER") + if self.options.verbose: + message_handler("Fetching allowed actions...", "HEADER") iam_client = self.options.client("iam") view_only_document = self.get_policy_allowed_calls( iam_client, "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess" @@ -580,9 +628,10 @@ def get_policies_allowed_actions(self): allowed_actions[action] = True for action in ON_TOP_POLICIES: allowed_actions[action] = True - message_handler( - "Found {} allowed actions".format(len(allowed_actions)), "HEADER" - ) + if self.options.verbose: + message_handler( + "Found {} allowed actions".format(len(allowed_actions)), "HEADER" + ) return allowed_actions.keys() diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 0b0332d..3975a7d 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -4,7 +4,7 @@ import functools import threading from abc import ABC -from typing import NamedTuple, List +from typing import NamedTuple, List, Dict from diskcache import Cache @@ -73,7 +73,8 @@ class Resource(NamedTuple): details: str = "" group: str = "" tags: List[ResourceTag] = [] - limits: List[LimitsValues] = [] + limits: LimitsValues = None + attributes: Dict[str, object] = {} class ResourceCache: diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 9030ab9..c3c040d 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -35,24 +35,46 @@ def general_report( + "%" ) # pylint: disable=line-too-long - message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 - resource.limits.service, - resource.limits.quota_code, - resource.limits.quota_name, - resource.limits.aws_limit, - resource.limits.local_limit, - usage, + message_handler( + "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 + resource.limits.service, + resource.limits.quota_code, + resource.limits.quota_name, + resource.limits.aws_limit, + resource.limits.local_limit, + usage, + ), + "OKBLUE", ) + elif resource.attributes: + message_handler( + "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + resource.digest.type, + resource.digest.id, + resource.name, + resource.details, + ), + "OKBLUE", + ) + for ( + resource_attr_key, + resource_attr_value, + ) in resource.attributes.items(): + message_handler( + "{}: {}".format(resource_attr_key, resource_attr_value,), + "OKBLUE", + ) else: - message = "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( - resource.digest.type, - resource.digest.id, - resource.name, - resource.details, + message_handler( + "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + resource.digest.type, + resource.digest.id, + resource.name, + resource.details, + ), + "OKBLUE", ) - message_handler(message, "OKBLUE") - if resource_relations: message_handler("\n\nFound relations", "HEADER") for resource_relation in resource_relations: @@ -88,17 +110,21 @@ def html_report( with open(image_name, "rb") as image_file: diagram_image = base64.b64encode(image_file.read()).decode("utf-8") + group_title = "Group" if resources: if resources[0].limits: html_output = dir_template.get_template("report_limits.html").render( default_name=title, resources_found=resources ) else: + if resources[0].attributes: + group_title = "Service" html_output = dir_template.get_template("report_html.html").render( default_name=title, resources_found=resources, resources_relations=resource_relations, diagram_image=diagram_image, + group_title=group_title, ) self.make_directories() diff --git a/cloudiscovery/templates/report_html.html b/cloudiscovery/templates/report_html.html index ed2cdec..2945f76 100644 --- a/cloudiscovery/templates/report_html.html +++ b/cloudiscovery/templates/report_html.html @@ -5,7 +5,7 @@ Type -Group +{{ group_title }} Id Name Details @@ -17,7 +17,12 @@ {{ resource_found.group}} {{ resource_found.digest.id}} {{ resource_found.name}} - {{ resource_found.details}} + + {{ resource_found.details}} + {% for attribute_key, attribute_value in resource_found.attributes.items() %} + {{ attribute_key}}: {{ attribute_value}}
+ {%- endfor %} + {% for tag in resource_found.tags %} {{ tag.key}}: {{ tag.value}}
From c74c512bf3952efefdcefa342796f616863677fe Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 14:58:03 +0700 Subject: [PATCH 77/86] simplified console reporting --- cloudiscovery/shared/report.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index c3c040d..977517e 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -47,8 +47,10 @@ def general_report( "OKBLUE", ) elif resource.attributes: + # pylint: disable=too-many-format-args message_handler( - "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + "\nservice: {} - type: {} - id: {} - resource name: {}".format( + resource.group, resource.digest.type, resource.digest.id, resource.name, @@ -61,12 +63,18 @@ def general_report( resource_attr_value, ) in resource.attributes.items(): message_handler( - "{}: {}".format(resource_attr_key, resource_attr_value,), + "service: {} - type: {} - id: {} -> {}: {}".format( + resource.group, + resource.digest.type, + resource.digest.id, + resource_attr_key, + resource_attr_value, + ), "OKBLUE", ) else: message_handler( - "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + "type: {} - id: {} - name: {} - details: {}".format( resource.digest.type, resource.digest.id, resource.name, @@ -78,7 +86,7 @@ def general_report( if resource_relations: message_handler("\n\nFound relations", "HEADER") for resource_relation in resource_relations: - message = "resource type: {} - resource id: {} -> resource type: {} - resource id: {}".format( + message = "type: {} - id: {} -> type: {} - id: {}".format( resource_relation.from_node.type, resource_relation.from_node.id, resource_relation.to_node.type, From 78dcd53bb2464d7455ec018872d8f91954e6cc05 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 15:33:45 +0700 Subject: [PATCH 78/86] silence aws_ec2_prefix_list --- cloudiscovery/provider/all/resource/all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 070921b..e412fe3 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -106,6 +106,7 @@ "aws_organizations_account", "aws_config_organization_config_rule_status", "aws_dynamodb_backup", + "aws_ec2_prefix_list", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS From c620a714b97a75fe20f7377906c5ecacc8c7b49a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:29:34 +0100 Subject: [PATCH 79/86] New limits check and filter param added --- cloudiscovery/provider/all/resource/all.py | 1 + cloudiscovery/provider/limit/command.py | 44 +++++++++++++++++++- cloudiscovery/provider/limit/resource/all.py | 13 +++++- cloudiscovery/shared/common_aws.py | 7 +++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index e412fe3..cbeccf1 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -555,6 +555,7 @@ def analyze_operation( client=client, operation_name=snake_operation_name, resource_type=resource_type, + filter=None, ) list_metadata = pages.result_keys[0].parsed result_key = None diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index d26fc5b..f7e7436 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -58,10 +58,48 @@ "global": False, }, "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": [],}, + "L-0485CB21": { + "method": "list_stacks", + "key": "StackSummaries", + "fields": [], + "filter": { + "StackStatusFilter": [ + "CREATE_IN_PROGRESS", + "CREATE_FAILED", + "CREATE_COMPLETE", + "ROLLBACK_IN_PROGRESS", + "ROLLBACK_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_IN_PROGRESS", + "DELETE_FAILED", + "UPDATE_IN_PROGRESS", + "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_COMPLETE", + "UPDATE_ROLLBACK_IN_PROGRESS", + "UPDATE_ROLLBACK_FAILED", + "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_ROLLBACK_COMPLETE", + "REVIEW_IN_PROGRESS", + "IMPORT_IN_PROGRESS", + "IMPORT_COMPLETE", + "IMPORT_ROLLBACK_IN_PROGRESS", + "IMPORT_ROLLBACK_FAILED", + "IMPORT_ROLLBACK_COMPLETE", + ] + }, + }, "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, "global": False, }, + "codeguru-reviewer": { + "L-F5129FC6": { + "method": "list_code_reviews", + "key": "CodeReviewSummaries", + "fields": [], + "filter": {"Type": "PullRequest"}, + }, + "global": False, + }, "dynamodb": { "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, "global": False, @@ -241,6 +279,10 @@ }, "global": False, }, + "vpc": { + "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, + "global": False, + }, } diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index c53bae0..83c70cf 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -17,6 +17,7 @@ SERVICEQUOTA_TO_BOTO3 = { "elasticloadbalancing": "elbv2", "elasticfilesystem": "efs", + "vpc": "ec2", } MAX_EXECUTION_PARALLEL = 3 @@ -123,13 +124,23 @@ def analyze_detail(self, client_quota, data_resource, service, threshold_request usage = 0 + # Check filters by resource + if "filter" in quota_data: + filters = quota_data["filter"] + else: + filters = None + pages = get_paginator( client=client, operation_name=quota_data["method"], resource_type="aws_limit", + filters=filters, ) if not pages: - response = getattr(client, quota_data["method"])() + if filters: + response = getattr(client, quota_data["method"])(**filters) + else: + response = getattr(client, quota_data["method"])() usage = len(response[quota_data["key"]]) else: for page in pages: diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index f4aa2a3..02eb2b9 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -242,7 +242,7 @@ def generate_session(profile_name): exit_critical(message) -def get_paginator(client, operation_name, resource_type): +def get_paginator(client, operation_name, resource_type, filters=None): # Checking if can paginate if client.can_paginate(operation_name): paginator = client.get_paginator(operation_name) @@ -251,7 +251,10 @@ def get_paginator(client, operation_name, resource_type): Scope="Local" ) # hack to list only local IAM policies - aws_all else: - pages = paginator.paginate() + if filters: + pages = paginator.paginate(**filters) + else: + pages = paginator.paginate() else: return False From 37497f94984634465b1071373ba948720d5ae6cd Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:30:55 +0100 Subject: [PATCH 80/86] Fix aws all --- cloudiscovery/provider/all/resource/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index cbeccf1..34967fe 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -555,7 +555,7 @@ def analyze_operation( client=client, operation_name=snake_operation_name, resource_type=resource_type, - filter=None, + filters=None, ) list_metadata = pages.result_keys[0].parsed result_key = None From 540d4022fa69c827e1ba5946fce0eabf35b02e83 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:33:31 +0100 Subject: [PATCH 81/86] Readme update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 26ac53e..64064eb 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,7 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * batch * codebuild * codecommit + * codeguru reviewer * cloudformation * dynamodb * ec2 @@ -318,6 +319,7 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * sns * transcribe * translate + * vpc AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. An administrator can ask to increase the quota value of a certain service via ticket. This command helps administrators detect those issues in advance. From e014f13c186592cf3da37b4935f54cbd39203976 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 18:40:59 +0100 Subject: [PATCH 82/86] New release --- cloudiscovery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index b9ac67b..bb5884f 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -46,7 +46,7 @@ print("Python 3.6 or newer is required", file=sys.stderr) sys.exit(1) -__version__ = "2.1.1" +__version__ = "2.2.0" AVAILABLE_LANGUAGES = ["en_US", "pt_BR"] DEFAULT_REGION = "us-east-1" From 8c2e0cdb03f140c9dc409f2d344cd34492f0888c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 00:18:47 +0100 Subject: [PATCH 83/86] Added more limit resources --- README.md | 4 ++ cloudiscovery/provider/limit/command.py | 60 ++++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 2 + 3 files changed, 66 insertions(+) diff --git a/README.md b/README.md index 64064eb..cacf6c1 100644 --- a/README.md +++ b/README.md @@ -295,16 +295,20 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * appsync * autoscaling-plans * batch + * chime * codebuild * codecommit * codeguru reviewer + * codeguru profiler * cloudformation + * cloud map * dynamodb * ec2 * ecs * elasticfilesystem * elasticbeanstalk * elasticloadbalancing + * glue * iam * kms * mediaconnect diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index f7e7436..160566b 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -37,6 +37,10 @@ }, "global": False, }, + "AWSCloudMap": { + "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, + "global": False, + }, "batch": { "L-144F0CA5": { "method": "describe_compute_environments", @@ -45,6 +49,24 @@ }, "global": False, }, + "chime": { + "L-8EE806B4": { + "method": "list_voice_connectors", + "key": "VoiceConnectors", + "fields": [], + }, + "L-32405DBA": { + "method": "list_phone_numbers", + "key": "PhoneNumbers", + "fields": [], + }, + "L-D3615084": { + "method": "list_voice_connector_groups", + "key": "VoiceConnectorGroups", + "fields": [], + }, + "global": True, + }, "codebuild": { "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, "global": False, @@ -89,6 +111,7 @@ }, }, "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, + "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, "global": False, }, "codeguru-reviewer": { @@ -100,6 +123,14 @@ }, "global": False, }, + "codeguru-profiler": { + "L-DA8D4E8D": { + "method": "list_profiling_groups", + "key": "profilingGroupNames", + "fields": [], + }, + "global": False, + }, "dynamodb": { "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, "global": False, @@ -135,6 +166,11 @@ "key": "Applications", "fields": [], }, + "L-D64F1F14": { + "method": "describe_application_versions", + "key": "ApplicationVersions", + "fields": [], + }, "global": False, }, "elasticloadbalancing": { @@ -145,6 +181,29 @@ }, "global": False, }, + "glue": { + "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, + "L-D987EC31": { + "method": "get_user_defined_functions", + "key": "UserDefinedFunctions", + "fields": [], + "filter": {"Pattern": "*"}, + }, + "L-83192DBF": { + "method": "get_security_configurations", + "key": "SecurityConfigurations", + "fields": [], + }, + "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, + "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, + "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, + "L-04CEE988": { + "method": "list_ml_transforms", + "key": "TransformIds", + "fields": [], + }, + "global": False, + }, "iam": { "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, @@ -169,6 +228,7 @@ }, "kms": { "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, "global": False, }, "mediaconnect": { diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 83c70cf..b713ce5 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -18,6 +18,8 @@ "elasticloadbalancing": "elbv2", "elasticfilesystem": "efs", "vpc": "ec2", + "codeguru-profiler": "codeguruprofiler", + "AWSCloudMap": "servicediscovery", } MAX_EXECUTION_PARALLEL = 3 From 909f8757e4bd3debf73e73bf04e161a9460cb47f Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 00:58:45 +0100 Subject: [PATCH 84/86] Added EC2 check --- cloudiscovery/provider/limit/command.py | 77 ++++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 5 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 160566b..bd32e00 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -141,6 +141,83 @@ "key": "Addresses", "fields": [], }, + "L-74FC7D96": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], + } + ] + }, + }, + "L-DB2E81BA": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "g3s.xlarge", + "g3.4xlarge", + "g3.8xlarge", + "g3.16xlarge", + "g4dn.xlarge", + "g4dn.2xlarge", + "g4dn.4xlarge", + "g4dn.8xlarge", + "g4dn.16xlarge", + "g4dn.12xlarge", + "g4dn.metal", + ], + } + ] + }, + }, + "L-1945791B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "inf1.xlarge", + "inf1.2xlarge", + "inf1.6xlarge", + "inf1.24xlarge", + ], + } + ] + }, + }, + "L-417A185B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "p2.xlarge", + "p2.8xlarge", + "p2.16xlarge", + "p3.2xlarge", + "p3.8xlarge", + "p3.16xlarge", + "p3dn.24xlarge", + ], + } + ] + }, + }, "global": False, }, "ecs": { diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index b713ce5..ee8fd54 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -148,7 +148,10 @@ def analyze_detail(self, client_quota, data_resource, service, threshold_request for page in pages: usage = usage + len(page[quota_data["key"]]) - percent = round((usage / value) * 100, 2) + try: + percent = round((usage / value) * 100, 2) + except ZeroDivisionError: + percent = 0 if percent >= threshold_requested: resources_found.append( From 36c5353a01b4b76c600222ee098064991f39e02a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 01:40:26 +0100 Subject: [PATCH 85/86] Added EC2 resources --- cloudiscovery/provider/limit/command.py | 419 +---------- .../provider/limit/data/allowed_resources.py | 651 ++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 3 +- 3 files changed, 654 insertions(+), 419 deletions(-) create mode 100644 cloudiscovery/provider/limit/data/allowed_resources.py diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index bd32e00..9bd9693 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -3,424 +3,7 @@ from shared.common import ResourceCache, message_handler, Filterable, BaseOptions from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram - -ALLOWED_SERVICES_CODES = { - "acm": { - "L-F141DD1D": { - "method": "list_certificates", - "key": "CertificateSummaryList", - "fields": [], - }, - "global": False, - }, - "amplify": { - "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, - "global": False, - }, - "appmesh": { - "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, - "global": False, - }, - "appsync": { - "L-06A0647C": { - "method": "list_graphql_apis", - "key": "graphqlApis", - "fields": [], - }, - "global": False, - }, - "autoscaling-plans": { - "L-BD401546": { - "method": "describe_scaling_plans", - "key": "ScalingPlans", - "fields": [], - }, - "global": False, - }, - "AWSCloudMap": { - "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, - "global": False, - }, - "batch": { - "L-144F0CA5": { - "method": "describe_compute_environments", - "key": "computeEnvironments", - "fields": [], - }, - "global": False, - }, - "chime": { - "L-8EE806B4": { - "method": "list_voice_connectors", - "key": "VoiceConnectors", - "fields": [], - }, - "L-32405DBA": { - "method": "list_phone_numbers", - "key": "PhoneNumbers", - "fields": [], - }, - "L-D3615084": { - "method": "list_voice_connector_groups", - "key": "VoiceConnectorGroups", - "fields": [], - }, - "global": True, - }, - "codebuild": { - "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, - "global": False, - }, - "codecommit": { - "L-81790602": { - "method": "list_repositories", - "key": "repositories", - "fields": [], - }, - "global": False, - }, - "cloudformation": { - "L-0485CB21": { - "method": "list_stacks", - "key": "StackSummaries", - "fields": [], - "filter": { - "StackStatusFilter": [ - "CREATE_IN_PROGRESS", - "CREATE_FAILED", - "CREATE_COMPLETE", - "ROLLBACK_IN_PROGRESS", - "ROLLBACK_FAILED", - "ROLLBACK_COMPLETE", - "DELETE_IN_PROGRESS", - "DELETE_FAILED", - "UPDATE_IN_PROGRESS", - "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", - "UPDATE_COMPLETE", - "UPDATE_ROLLBACK_IN_PROGRESS", - "UPDATE_ROLLBACK_FAILED", - "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", - "UPDATE_ROLLBACK_COMPLETE", - "REVIEW_IN_PROGRESS", - "IMPORT_IN_PROGRESS", - "IMPORT_COMPLETE", - "IMPORT_ROLLBACK_IN_PROGRESS", - "IMPORT_ROLLBACK_FAILED", - "IMPORT_ROLLBACK_COMPLETE", - ] - }, - }, - "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, - "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, - "global": False, - }, - "codeguru-reviewer": { - "L-F5129FC6": { - "method": "list_code_reviews", - "key": "CodeReviewSummaries", - "fields": [], - "filter": {"Type": "PullRequest"}, - }, - "global": False, - }, - "codeguru-profiler": { - "L-DA8D4E8D": { - "method": "list_profiling_groups", - "key": "profilingGroupNames", - "fields": [], - }, - "global": False, - }, - "dynamodb": { - "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, - "global": False, - }, - "ec2": { - "L-0263D0A3": { - "method": "describe_addresses", - "key": "Addresses", - "fields": [], - }, - "L-74FC7D96": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], - } - ] - }, - }, - "L-DB2E81BA": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "g3s.xlarge", - "g3.4xlarge", - "g3.8xlarge", - "g3.16xlarge", - "g4dn.xlarge", - "g4dn.2xlarge", - "g4dn.4xlarge", - "g4dn.8xlarge", - "g4dn.16xlarge", - "g4dn.12xlarge", - "g4dn.metal", - ], - } - ] - }, - }, - "L-1945791B": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "inf1.xlarge", - "inf1.2xlarge", - "inf1.6xlarge", - "inf1.24xlarge", - ], - } - ] - }, - }, - "L-417A185B": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "p2.xlarge", - "p2.8xlarge", - "p2.16xlarge", - "p3.2xlarge", - "p3.8xlarge", - "p3.16xlarge", - "p3dn.24xlarge", - ], - } - ] - }, - }, - "global": False, - }, - "ecs": { - "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, - "global": False, - }, - "elasticfilesystem": { - "L-848C634D": { - "method": "describe_file_systems", - "key": "FileSystems", - "fields": [], - }, - "global": False, - }, - "elasticbeanstalk": { - "L-8EFC1C51": { - "method": "describe_environments", - "key": "Environments", - "fields": [], - }, - "L-1CEABD17": { - "method": "describe_applications", - "key": "Applications", - "fields": [], - }, - "L-D64F1F14": { - "method": "describe_application_versions", - "key": "ApplicationVersions", - "fields": [], - }, - "global": False, - }, - "elasticloadbalancing": { - "L-53DA6B97": { - "method": "describe_load_balancers", - "key": "LoadBalancers", - "fields": [], - }, - "global": False, - }, - "glue": { - "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, - "L-D987EC31": { - "method": "get_user_defined_functions", - "key": "UserDefinedFunctions", - "fields": [], - "filter": {"Pattern": "*"}, - }, - "L-83192DBF": { - "method": "get_security_configurations", - "key": "SecurityConfigurations", - "fields": [], - }, - "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, - "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, - "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, - "L-04CEE988": { - "method": "list_ml_transforms", - "key": "TransformIds", - "fields": [], - }, - "global": False, - }, - "iam": { - "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, - "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, - "L-BF35879D": { - "method": "list_server_certificates", - "key": "ServerCertificateMetadataList", - "fields": [], - }, - "L-6E65F664": { - "method": "list_instance_profiles", - "key": "InstanceProfiles", - "fields": [], - "paginate": False, - }, - "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, - "L-DB618D39": { - "method": "list_saml_providers", - "key": "SAMLProviderList", - "fields": [], - }, - "global": True, - }, - "kms": { - "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, - "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, - "global": False, - }, - "mediaconnect": { - "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, - "L-F1F62F5D": { - "method": "list_entitlements", - "key": "Entitlements", - "fields": [], - }, - "global": False, - }, - "medialive": { - "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, - "L-BDF24E14": { - "method": "list_input_devices", - "key": "InputDevices", - "fields": [], - }, - "global": False, - }, - "mediapackage": { - "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, - "global": False, - }, - "qldb": { - "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, - "global": False, - }, - "robomaker": { - "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, - "L-D6554FB1": { - "method": "list_simulation_applications", - "key": "simulationApplicationSummaries", - "fields": [], - }, - "global": False, - }, - "route53": { - "L-4EA4796A": { - "method": "list_hosted_zones", - "key": "HostedZones", - "fields": [], - }, - "L-ACB674F3": { - "method": "list_health_checks", - "key": "HealthChecks", - "fields": [], - }, - "global": True, - }, - "route53resolver": { - "L-4A669CC0": { - "method": "list_resolver_endpoints", - "key": "ResolverEndpoints", - "fields": [], - }, - "L-51D8A1FB": { - "method": "list_resolver_rules", - "key": "ResolverRules", - "fields": [], - }, - "global": True, - }, - "rds": { - "L-7B6409FD": { - "method": "describe_db_instances", - "key": "DBInstances", - "fields": [], - }, - "L-952B80B8": { - "method": "describe_db_clusters", - "key": "DBClusters", - "fields": [], - }, - "L-DE55804A": { - "method": "describe_db_parameter_groups", - "key": "DBParameterGroups", - "fields": [], - }, - "L-9FA33840": { - "method": "describe_option_groups", - "key": "OptionGroupsList", - "fields": [], - }, - "global": False, - }, - "s3": { - "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, - "global": False, - }, - "sns": { - "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, - "global": False, - }, - "transcribe": { - "L-3278D334": { - "method": "list_vocabularies", - "key": "Vocabularies", - "fields": [], - }, - "global": False, - }, - "translate": { - "L-4011ABD8": { - "method": "list_terminologies", - "key": "TerminologyPropertiesList", - "fields": [], - }, - "global": False, - }, - "vpc": { - "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, - "global": False, - }, -} +from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES class LimitOptions(BaseAwsOptions, BaseOptions): diff --git a/cloudiscovery/provider/limit/data/allowed_resources.py b/cloudiscovery/provider/limit/data/allowed_resources.py new file mode 100644 index 0000000..25f1aa7 --- /dev/null +++ b/cloudiscovery/provider/limit/data/allowed_resources.py @@ -0,0 +1,651 @@ +ALLOWED_SERVICES_CODES = { + "acm": { + "L-F141DD1D": { + "method": "list_certificates", + "key": "CertificateSummaryList", + "fields": [], + }, + "global": False, + }, + "amplify": { + "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, + "global": False, + }, + "appmesh": { + "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, + "global": False, + }, + "appsync": { + "L-06A0647C": { + "method": "list_graphql_apis", + "key": "graphqlApis", + "fields": [], + }, + "global": False, + }, + "autoscaling-plans": { + "L-BD401546": { + "method": "describe_scaling_plans", + "key": "ScalingPlans", + "fields": [], + }, + "global": False, + }, + "AWSCloudMap": { + "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, + "global": False, + }, + "batch": { + "L-144F0CA5": { + "method": "describe_compute_environments", + "key": "computeEnvironments", + "fields": [], + }, + "global": False, + }, + "chime": { + "L-8EE806B4": { + "method": "list_voice_connectors", + "key": "VoiceConnectors", + "fields": [], + }, + "L-32405DBA": { + "method": "list_phone_numbers", + "key": "PhoneNumbers", + "fields": [], + }, + "L-D3615084": { + "method": "list_voice_connector_groups", + "key": "VoiceConnectorGroups", + "fields": [], + }, + "global": True, + }, + "codebuild": { + "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, + "global": False, + }, + "codecommit": { + "L-81790602": { + "method": "list_repositories", + "key": "repositories", + "fields": [], + }, + "global": False, + }, + "cloudformation": { + "L-0485CB21": { + "method": "list_stacks", + "key": "StackSummaries", + "fields": [], + "filter": { + "StackStatusFilter": [ + "CREATE_IN_PROGRESS", + "CREATE_FAILED", + "CREATE_COMPLETE", + "ROLLBACK_IN_PROGRESS", + "ROLLBACK_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_IN_PROGRESS", + "DELETE_FAILED", + "UPDATE_IN_PROGRESS", + "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_COMPLETE", + "UPDATE_ROLLBACK_IN_PROGRESS", + "UPDATE_ROLLBACK_FAILED", + "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_ROLLBACK_COMPLETE", + "REVIEW_IN_PROGRESS", + "IMPORT_IN_PROGRESS", + "IMPORT_COMPLETE", + "IMPORT_ROLLBACK_IN_PROGRESS", + "IMPORT_ROLLBACK_FAILED", + "IMPORT_ROLLBACK_COMPLETE", + ] + }, + }, + "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, + "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, + "global": False, + }, + "codeguru-reviewer": { + "L-F5129FC6": { + "method": "list_code_reviews", + "key": "CodeReviewSummaries", + "fields": [], + "filter": {"Type": "PullRequest"}, + }, + "global": False, + }, + "codeguru-profiler": { + "L-DA8D4E8D": { + "method": "list_profiling_groups", + "key": "profilingGroupNames", + "fields": [], + }, + "global": False, + }, + "dynamodb": { + "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, + "global": False, + }, + "ec2": { + "L-0263D0A3": { + "method": "describe_addresses", + "key": "Addresses", + "fields": [], + }, + "L-74FC7D96": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], + } + ] + }, + }, + "L-DB2E81BA": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "g3s.xlarge", + "g3.4xlarge", + "g3.8xlarge", + "g3.16xlarge", + "g4dn.xlarge", + "g4dn.2xlarge", + "g4dn.4xlarge", + "g4dn.8xlarge", + "g4dn.16xlarge", + "g4dn.12xlarge", + "g4dn.metal", + ], + } + ] + }, + }, + "L-1945791B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "inf1.xlarge", + "inf1.2xlarge", + "inf1.6xlarge", + "inf1.24xlarge", + ], + } + ] + }, + }, + "L-417A185B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "p2.xlarge", + "p2.8xlarge", + "p2.16xlarge", + "p3.2xlarge", + "p3.8xlarge", + "p3.16xlarge", + "p3dn.24xlarge", + ], + } + ] + }, + }, + "L-1216C47A": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "c5d.large", + "c5d.xlarge", + "c5d.2xlarge", + "c5d.4xlarge", + "c5d.9xlarge", + "c5d.12xlarge", + "c5d.18xlarge", + "c5d.24xlarge", + "c5d.metal", + "c5a.large", + "c5a.xlarge", + "c5a.2xlarge", + "c5a.4xlarge", + "c5a.8xlarge", + "c5a.12xlarge", + "c5a.16xlarge", + "c5a.24xlarge", + "c5n.large", + "c5n.xlarge", + "c5n.2xlarge", + "c5n.4xlarge", + "c5n.9xlarge", + "c5n.18xlarge", + "c5n.metal", + "c4.large", + "c4.xlarge", + "c4.2xlarge", + "c4.4xlarge", + "c4.8xlarge", + "d2.xlarge", + "d2.2xlarge", + "d2.4xlarge", + "d2.8xlarge", + "h1.2xlarge", + "h1.4xlarge", + "h1.8xlarge", + "h1.16xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", + "i3.metal", + "m6g.medium", + "m6g.large", + "m6g.xlarge", + "m6g.2xlarge", + "m6g.4xlarge", + "m6g.8xlarge", + "m6g.12xlarge", + "m6g.16xlarge", + "m6g.metal", + "m5.large", + "m5.xlarge", + "m5.2xlarge", + "m5.4xlarge", + "m5.8xlarge", + "m5.12xlarge", + "m5.16xlarge", + "m5.24xlarge", + "m5.metal", + "m5d.large", + "m5d.xlarge", + "m5d.2xlarge", + "m5d.4xlarge", + "m5d.8xlarge", + "m5d.12xlarge", + "m5d.16xlarge", + "m5d.24xlarge", + "m5d.metal", + "m5a.large", + "m5a.xlarge", + "m5a.2xlarge", + "m5a.4xlarge", + "m5a.8xlarge", + "m5a.12xlarge", + "m5a.16xlarge", + "m5a.24xlarge", + "m5ad.large", + "m5ad.xlarge", + "m5ad.2xlarge", + "m5ad.4xlarge", + "m5ad.12xlarge", + "m5ad.24xlarge", + "m5n.large", + "m5n.xlarge", + "m5n.2xlarge", + "m5n.4xlarge", + "m5n.8xlarge", + "m5n.12xlarge", + "m5n.16xlarge", + "m5n.24xlarge", + "m5dn.large", + "m5dn.xlarge", + "m5dn.2xlarge", + "m5dn.4xlarge", + "m5dn.8xlarge", + "m5dn.12xlarge", + "m5dn.16xlarge", + "m5dn.24xlarge", + "m4.large", + "m4.xlarge", + "m4.2xlarge", + "m4.4xlarge", + "m4.10xlarge", + "m4.16xlarge", + "z1d.large", + "z1d.xlarge", + "z1d.2xlarge", + "z1d.3xlarge", + "z1d.6xlarge", + "z1d.12xlarge", + "z1d.metal", + "r6g.medium", + "r6g.large", + "r6g.xlarge", + "r6g.2xlarge", + "r6g.4xlarge", + "r6g.8xlarge", + "r6g.12xlarge", + "r6g.16xlarge", + "r6g.metal", + "r5.large", + "r5.xlarge", + "r5.2xlarge", + "r5.4xlarge", + "r5.8xlarge", + "r5.12xlarge", + "r5.16xlarge", + "r5.24xlarge", + "r5.metal", + "r5d.large", + "r5d.xlarge", + "r5d.2xlarge", + "r5d.4xlarge", + "r5d.8xlarge", + "r5d.12xlarge", + "r5d.16xlarge", + "r5d.24xlarge", + "r5d.metal", + "r5a.large", + "r5a.xlarge", + "r5a.2xlarge", + "r5a.4xlarge", + "r5a.8xlarge", + "r5a.12xlarge", + "r5a.16xlarge", + "r5a.24xlarge", + "r5ad.large", + "r5ad.xlarge", + "r5ad.2xlarge", + "r5ad.4xlarge", + "r5ad.12xlarge", + "r5ad.24xlarge", + "r5n.large", + "r5n.xlarge", + "r5n.2xlarge", + "r5n.4xlarge", + "r5n.8xlarge", + "r5n.12xlarge", + "r5n.16xlarge", + "r5n.24xlarge", + "r5dn.large", + "r5dn.xlarge", + "r5dn.2xlarge", + "r5dn.4xlarge", + "r5dn.8xlarge", + "r5dn.12xlarge", + "r5dn.16xlarge", + "r5dn.24xlarge", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", + "t3.nano", + "t3.micro", + "t3.small", + "t3.medium", + "t3.large", + "t3.xlarge", + "t3.2xlarge", + "t3a.nano", + "t3a.micro", + "t3a.small", + "t3a.medium", + "t3a.large", + "t3a.xlarge", + "t3a.2xlarge", + "t2.nano", + "t2.micro", + "t2.small", + "t2.medium", + "t2.large", + "t2.xlarge", + "t2.2xlarge", + ], + } + ] + }, + }, + "L-7295265B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "x1e.xlarge", + "x1e.2xlarge", + "x1e.4xlarge", + "x1e.8xlarge", + "x1e.16xlarge", + "x1e.32xlarge", + "x1.16xlarge", + "x1.32xlarge", + ], + } + ] + }, + }, + "global": False, + }, + "ecs": { + "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, + "global": False, + }, + "elasticfilesystem": { + "L-848C634D": { + "method": "describe_file_systems", + "key": "FileSystems", + "fields": [], + }, + "global": False, + }, + "elasticbeanstalk": { + "L-8EFC1C51": { + "method": "describe_environments", + "key": "Environments", + "fields": [], + }, + "L-1CEABD17": { + "method": "describe_applications", + "key": "Applications", + "fields": [], + }, + "L-D64F1F14": { + "method": "describe_application_versions", + "key": "ApplicationVersions", + "fields": [], + }, + "global": False, + }, + "elasticloadbalancing": { + "L-53DA6B97": { + "method": "describe_load_balancers", + "key": "LoadBalancers", + "fields": [], + }, + "global": False, + }, + "glue": { + "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, + "L-D987EC31": { + "method": "get_user_defined_functions", + "key": "UserDefinedFunctions", + "fields": [], + "filter": {"Pattern": "*"}, + }, + "L-83192DBF": { + "method": "get_security_configurations", + "key": "SecurityConfigurations", + "fields": [], + }, + "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, + "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, + "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, + "L-04CEE988": { + "method": "list_ml_transforms", + "key": "TransformIds", + "fields": [], + }, + "global": False, + }, + "iam": { + "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, + "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, + "L-BF35879D": { + "method": "list_server_certificates", + "key": "ServerCertificateMetadataList", + "fields": [], + }, + "L-6E65F664": { + "method": "list_instance_profiles", + "key": "InstanceProfiles", + "fields": [], + "paginate": False, + }, + "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, + "L-DB618D39": { + "method": "list_saml_providers", + "key": "SAMLProviderList", + "fields": [], + }, + "global": True, + }, + "kms": { + "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, + "global": False, + }, + "mediaconnect": { + "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, + "L-F1F62F5D": { + "method": "list_entitlements", + "key": "Entitlements", + "fields": [], + }, + "global": False, + }, + "medialive": { + "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, + "L-BDF24E14": { + "method": "list_input_devices", + "key": "InputDevices", + "fields": [], + }, + "global": False, + }, + "mediapackage": { + "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, + "global": False, + }, + "qldb": { + "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, + "global": False, + }, + "robomaker": { + "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, + "L-D6554FB1": { + "method": "list_simulation_applications", + "key": "simulationApplicationSummaries", + "fields": [], + }, + "global": False, + }, + "route53": { + "L-4EA4796A": { + "method": "list_hosted_zones", + "key": "HostedZones", + "fields": [], + }, + "L-ACB674F3": { + "method": "list_health_checks", + "key": "HealthChecks", + "fields": [], + }, + "global": True, + }, + "route53resolver": { + "L-4A669CC0": { + "method": "list_resolver_endpoints", + "key": "ResolverEndpoints", + "fields": [], + }, + "L-51D8A1FB": { + "method": "list_resolver_rules", + "key": "ResolverRules", + "fields": [], + }, + "global": True, + }, + "rds": { + "L-7B6409FD": { + "method": "describe_db_instances", + "key": "DBInstances", + "fields": [], + }, + "L-952B80B8": { + "method": "describe_db_clusters", + "key": "DBClusters", + "fields": [], + }, + "L-DE55804A": { + "method": "describe_db_parameter_groups", + "key": "DBParameterGroups", + "fields": [], + }, + "L-9FA33840": { + "method": "describe_option_groups", + "key": "OptionGroupsList", + "fields": [], + }, + "global": False, + }, + "s3": { + "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, + "global": False, + }, + "sns": { + "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, + "global": False, + }, + "transcribe": { + "L-3278D334": { + "method": "list_vocabularies", + "key": "Vocabularies", + "fields": [], + }, + "global": False, + }, + "translate": { + "L-4011ABD8": { + "method": "list_terminologies", + "key": "TerminologyPropertiesList", + "fields": [], + }, + "global": False, + }, + "vpc": { + "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, + "global": False, + }, +} diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index ee8fd54..b73c299 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -2,7 +2,8 @@ from concurrent.futures.thread import ThreadPoolExecutor -from provider.limit.command import LimitOptions, ALLOWED_SERVICES_CODES +from provider.limit.command import LimitOptions +from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES from shared.common import ( ResourceProvider, Resource, From c4041a40f65008072c22aa4f00ee1db4a29afd39 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 6 Jul 2020 14:10:34 +0700 Subject: [PATCH 86/86] limits - code review --- README.md | 16 +- cloudiscovery/provider/limit/command.py | 67 +++++-- cloudiscovery/provider/limit/resource/all.py | 192 ++++++++++--------- cloudiscovery/shared/error_handler.py | 8 +- 4 files changed, 163 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index cacf6c1..1e5f3f2 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,19 @@ The configured credentials must be associated to a user or role with proper perm "amplify:ListApps", "autoscaling-plans:DescribeScalingPlans", "medialive:ListChannels", + "medialive:ListInputDevices", "mediapackage:ListChannels", "qldb:ListLedgers", - "transcribe:ListVocabularies" + "transcribe:ListVocabularies", + "glue:GetDatabases", + "glue:GetUserDefinedFunctions", + "glue:GetSecurityConfigurations", + "glue:GetTriggers", + "glue:GetCrawlers", + "glue:ListWorkflows", + "glue:ListMLTransforms", + "codeguru-reviewer:ListCodeReviews", + "servicediscovery:ListNamespaces" ], "Resource": [ "*" ] } @@ -282,9 +292,9 @@ Types of resources mostly cover Terraform types. It is possible to narrow down s ### AWS Limit -It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. +It's possible to check resources limits across various service in an account. This command implements over 60 limits checks. -With `--services value,value,value` selection, you can narrow down checks to services that you want to check. +With `--services value,value,value` parameter, you can narrow down checks to just services that you want to check. With `--threshold 0-100` option, you can customize a minimum percentage threshold to start reporting a warning. diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 9bd9693..83f9bd1 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -1,6 +1,12 @@ from typing import List -from shared.common import ResourceCache, message_handler, Filterable, BaseOptions +from shared.common import ( + ResourceCache, + message_handler, + Filterable, + BaseOptions, + log_critical, +) from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES @@ -27,10 +33,11 @@ def __init__( class LimitParameters: - def __init__(self, session, region: str, services): + def __init__(self, session, region: str, services, options: LimitOptions): self.region = region self.cache = ResourceCache() self.session = session + self.options = options self.services = [] if services is None: for service in ALLOWED_SERVICES_CODES: @@ -52,12 +59,13 @@ def init_globalaws_limits_cache(self): if cache is not None: continue - message_handler( - "Fetching aws global limit to service {} in region {} to cache...".format( - service_code, self.region - ), - "HEADER", - ) + if self.options.verbose: + message_handler( + "Fetching aws global limit to service {} in region {} to cache...".format( + service_code, self.region + ), + "HEADER", + ) cache_codes = dict() for quota_code in ALLOWED_SERVICES_CODES[service_code]: @@ -76,16 +84,11 @@ def init_globalaws_limits_cache(self): "service-quotas", region_name=self.region ) - response = service_quota.get_aws_default_service_quota( - ServiceCode=service_code, QuotaCode=quota_code + item_to_add = self.get_quota( + quota_code, service_code, service_quota ) - - item_to_add = { - "value": response["Quota"]["Value"], - "adjustable": response["Quota"]["Adjustable"], - "quota_code": quota_code, - "quota_name": response["Quota"]["QuotaName"], - } + if item_to_add is None: + continue if service_code in cache_codes: cache_codes[service_code].append(item_to_add) @@ -96,6 +99,28 @@ def init_globalaws_limits_cache(self): return True + def get_quota(self, quota_code, service_code, service_quota): + try: + response = service_quota.get_aws_default_service_quota( + ServiceCode=service_code, QuotaCode=quota_code + ) + # pylint: disable=broad-except + except Exception as e: + if self.options.verbose: + log_critical( + "\nCannot take quota {} for {}: {}".format( + quota_code, service_code, str(e) + ) + ) + return None + item_to_add = { + "value": response["Quota"]["Value"], + "adjustable": response["Quota"]["Adjustable"], + "quota_code": quota_code, + "quota_name": response["Quota"]["QuotaName"], + } + return item_to_add + class Limit(BaseAwsCommand): def __init__(self, region_names, session, threshold): @@ -109,10 +134,10 @@ def __init__(self, region_names, session, threshold): super().__init__(region_names, session) self.threshold = threshold - def init_globalaws_limits_cache(self, region, services): + def init_globalaws_limits_cache(self, region, services, options: LimitOptions): # Cache services global and local services LimitParameters( - session=self.session, region=region, services=services + session=self.session, region=region, services=services, options=options ).init_globalaws_limits_cache() def run( @@ -128,7 +153,6 @@ def run( services.append(service) for region in self.region_names: - self.init_globalaws_limits_cache(region=region, services=services) limit_options = LimitOptions( verbose=verbose, filters=filters, @@ -137,6 +161,9 @@ def run( services=services, threshold=self.threshold, ) + self.init_globalaws_limits_cache( + region=region, services=services, options=limit_options + ) command_runner = AwsCommandRunner(services=services) command_runner.run( diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index b73c299..e2690d0 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -23,7 +23,7 @@ "AWSCloudMap": "servicediscovery", } -MAX_EXECUTION_PARALLEL = 3 +MAX_EXECUTION_PARALLEL = 2 class LimitResources(ResourceProvider): @@ -53,8 +53,8 @@ def get_resources(self) -> List[Resource]: with ThreadPoolExecutor(MAX_EXECUTION_PARALLEL) as executor: results = executor.map( - lambda aws_limit: self.analyze_service( - aws_limit=aws_limit, + lambda service_name: self.analyze_service( + service_name=service_name, client_quota=client_quota, threshold_requested=int(threshold_requested), ), @@ -68,110 +68,112 @@ def get_resources(self) -> List[Resource]: return resources_found @exception - def analyze_service(self, aws_limit, client_quota, threshold_requested): - - service = aws_limit - - cache_key = "aws_limits_" + service + "_" + self.options.region_name + def analyze_service(self, service_name, client_quota, threshold_requested): + cache_key = "aws_limits_" + service_name + "_" + self.options.region_name cache = self.cache.get_key(cache_key) - - return self.analyze_detail( - client_quota=client_quota, - data_resource=cache[service], - service=service, - threshold_requested=threshold_requested, - ) + resources_found = [] + if service_name not in cache: + return [] + + for data_quota_code in cache[service_name]: + if data_quota_code is None: + continue + resource_found = self.analyze_quota( + client_quota=client_quota, + data_quota_code=data_quota_code, + service=service_name, + threshold_requested=threshold_requested, + ) + if resource_found is not None: + resources_found.append(resource_found) + return resources_found @exception # pylint: disable=too-many-locals - def analyze_detail(self, client_quota, data_resource, service, threshold_requested): + def analyze_quota( + self, client_quota, data_quota_code, service, threshold_requested + ): + resource_found = None + quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] - resources_found = [] - - for data_quota_code in data_resource: - - quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] - - value_aws = value = data_quota_code["value"] + value_aws = value = data_quota_code["value"] - # Quota is adjustable by ticket request, then must override this values. - if bool(data_quota_code["adjustable"]) is True: - try: - response_quota = client_quota.get_service_quota( - ServiceCode=service, QuotaCode=data_quota_code["quota_code"] - ) - if "Value" in response_quota["Quota"]: - value = response_quota["Quota"]["Value"] - else: - value = data_quota_code["value"] - except client_quota.exceptions.NoSuchResourceException: + # Quota is adjustable by ticket request, then must override this values. + if bool(data_quota_code["adjustable"]) is True: + try: + response_quota = client_quota.get_service_quota( + ServiceCode=service, QuotaCode=data_quota_code["quota_code"] + ) + if "Value" in response_quota["Quota"]: + value = response_quota["Quota"]["Value"] + else: value = data_quota_code["value"] + except client_quota.exceptions.NoSuchResourceException: + value = data_quota_code["value"] + + if self.options.verbose: + message_handler( + "Collecting data from Quota: " + + service + + " - " + + data_quota_code["quota_name"] + + "...", + "HEADER", + ) - if self.options.verbose: - message_handler( - "Collecting data from Quota: " - + service - + " - " - + data_quota_code["quota_name"] - + "...", - "HEADER", - ) + # Need to convert some quota-services endpoint + if service in SERVICEQUOTA_TO_BOTO3: + service = SERVICEQUOTA_TO_BOTO3.get(service) - # Need to convert some quota-services endpoint - if service in SERVICEQUOTA_TO_BOTO3: - service = SERVICEQUOTA_TO_BOTO3.get(service) + client = self.options.session.client( + service, region_name=self.options.region_name + ) - client = self.options.session.client( - service, region_name=self.options.region_name - ) + usage = 0 - usage = 0 + # Check filters by resource + if "filter" in quota_data: + filters = quota_data["filter"] + else: + filters = None - # Check filters by resource - if "filter" in quota_data: - filters = quota_data["filter"] + pages = get_paginator( + client=client, + operation_name=quota_data["method"], + resource_type="aws_limit", + filters=filters, + ) + if not pages: + if filters: + response = getattr(client, quota_data["method"])(**filters) else: - filters = None - - pages = get_paginator( - client=client, - operation_name=quota_data["method"], - resource_type="aws_limit", - filters=filters, + response = getattr(client, quota_data["method"])() + usage = len(response[quota_data["key"]]) + else: + for page in pages: + usage = usage + len(page[quota_data["key"]]) + + try: + percent = round((usage / value) * 100, 2) + except ZeroDivisionError: + percent = 0 + + if percent >= threshold_requested: + resource_found = Resource( + digest=ResourceDigest( + id=data_quota_code["quota_code"], type="aws_limit" + ), + name="", + group="", + limits=LimitsValues( + quota_name=data_quota_code["quota_name"], + quota_code=data_quota_code["quota_code"], + aws_limit=int(value_aws), + local_limit=int(value), + usage=int(usage), + service=service, + percent=percent, + ), ) - if not pages: - if filters: - response = getattr(client, quota_data["method"])(**filters) - else: - response = getattr(client, quota_data["method"])() - usage = len(response[quota_data["key"]]) - else: - for page in pages: - usage = usage + len(page[quota_data["key"]]) - try: - percent = round((usage / value) * 100, 2) - except ZeroDivisionError: - percent = 0 - - if percent >= threshold_requested: - resources_found.append( - Resource( - digest=ResourceDigest( - id=data_quota_code["quota_code"], type="aws_limit" - ), - name="", - group="", - limits=LimitsValues( - quota_name=data_quota_code["quota_name"], - quota_code=data_quota_code["quota_code"], - aws_limit=int(value_aws), - local_limit=int(value), - usage=int(usage), - service=service, - percent=percent, - ), - ) - ) - - return resources_found + return resource_found diff --git a/cloudiscovery/shared/error_handler.py b/cloudiscovery/shared/error_handler.py index 0be51cd..835182f 100644 --- a/cloudiscovery/shared/error_handler.py +++ b/cloudiscovery/shared/error_handler.py @@ -11,13 +11,17 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) # pylint: disable=broad-except except Exception as e: - if "Could not connect to the endpoint URL" in str(e): + exception_str = str(e) + if ( + "Could not connect to the endpoint URL" in exception_str + or "the specified service does not exist" in exception_str + ): message = "\nThe service {} is not available in this region".format( func.__qualname__ ) else: message = "\nError running check {}. Error message {}".format( - func.__qualname__, str(e) + func.__qualname__, exception_str ) log_critical(message)