From c620a714b97a75fe20f7377906c5ecacc8c7b49a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:29:34 +0100 Subject: [PATCH 1/8] New limits check and filter param added --- cloudiscovery/provider/all/resource/all.py | 1 + cloudiscovery/provider/limit/command.py | 44 +++++++++++++++++++- cloudiscovery/provider/limit/resource/all.py | 13 +++++- cloudiscovery/shared/common_aws.py | 7 +++- 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index e412fe3..cbeccf1 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -555,6 +555,7 @@ def analyze_operation( client=client, operation_name=snake_operation_name, resource_type=resource_type, + filter=None, ) list_metadata = pages.result_keys[0].parsed result_key = None diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index d26fc5b..f7e7436 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -58,10 +58,48 @@ "global": False, }, "cloudformation": { - "L-0485CB21": {"method": "list_stacks", "key": "StackSummaries", "fields": [],}, + "L-0485CB21": { + "method": "list_stacks", + "key": "StackSummaries", + "fields": [], + "filter": { + "StackStatusFilter": [ + "CREATE_IN_PROGRESS", + "CREATE_FAILED", + "CREATE_COMPLETE", + "ROLLBACK_IN_PROGRESS", + "ROLLBACK_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_IN_PROGRESS", + "DELETE_FAILED", + "UPDATE_IN_PROGRESS", + "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_COMPLETE", + "UPDATE_ROLLBACK_IN_PROGRESS", + "UPDATE_ROLLBACK_FAILED", + "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_ROLLBACK_COMPLETE", + "REVIEW_IN_PROGRESS", + "IMPORT_IN_PROGRESS", + "IMPORT_COMPLETE", + "IMPORT_ROLLBACK_IN_PROGRESS", + "IMPORT_ROLLBACK_FAILED", + "IMPORT_ROLLBACK_COMPLETE", + ] + }, + }, "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, "global": False, }, + "codeguru-reviewer": { + "L-F5129FC6": { + "method": "list_code_reviews", + "key": "CodeReviewSummaries", + "fields": [], + "filter": {"Type": "PullRequest"}, + }, + "global": False, + }, "dynamodb": { "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, "global": False, @@ -241,6 +279,10 @@ }, "global": False, }, + "vpc": { + "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, + "global": False, + }, } diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index c53bae0..83c70cf 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -17,6 +17,7 @@ SERVICEQUOTA_TO_BOTO3 = { "elasticloadbalancing": "elbv2", "elasticfilesystem": "efs", + "vpc": "ec2", } MAX_EXECUTION_PARALLEL = 3 @@ -123,13 +124,23 @@ def analyze_detail(self, client_quota, data_resource, service, threshold_request usage = 0 + # Check filters by resource + if "filter" in quota_data: + filters = quota_data["filter"] + else: + filters = None + pages = get_paginator( client=client, operation_name=quota_data["method"], resource_type="aws_limit", + filters=filters, ) if not pages: - response = getattr(client, quota_data["method"])() + if filters: + response = getattr(client, quota_data["method"])(**filters) + else: + response = getattr(client, quota_data["method"])() usage = len(response[quota_data["key"]]) else: for page in pages: diff --git a/cloudiscovery/shared/common_aws.py b/cloudiscovery/shared/common_aws.py index f4aa2a3..02eb2b9 100644 --- a/cloudiscovery/shared/common_aws.py +++ b/cloudiscovery/shared/common_aws.py @@ -242,7 +242,7 @@ def generate_session(profile_name): exit_critical(message) -def get_paginator(client, operation_name, resource_type): +def get_paginator(client, operation_name, resource_type, filters=None): # Checking if can paginate if client.can_paginate(operation_name): paginator = client.get_paginator(operation_name) @@ -251,7 +251,10 @@ def get_paginator(client, operation_name, resource_type): Scope="Local" ) # hack to list only local IAM policies - aws_all else: - pages = paginator.paginate() + if filters: + pages = paginator.paginate(**filters) + else: + pages = paginator.paginate() else: return False From 37497f94984634465b1071373ba948720d5ae6cd Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:30:55 +0100 Subject: [PATCH 2/8] Fix aws all --- cloudiscovery/provider/all/resource/all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index cbeccf1..34967fe 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -555,7 +555,7 @@ def analyze_operation( client=client, operation_name=snake_operation_name, resource_type=resource_type, - filter=None, + filters=None, ) list_metadata = pages.result_keys[0].parsed result_key = None From 540d4022fa69c827e1ba5946fce0eabf35b02e83 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 16:33:31 +0100 Subject: [PATCH 3/8] Readme update --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 26ac53e..64064eb 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,7 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * batch * codebuild * codecommit + * codeguru reviewer * cloudformation * dynamodb * ec2 @@ -318,6 +319,7 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * sns * transcribe * translate + * vpc AWS has a default quota to all services. At the first time that an account is created, AWS apply this default quota to all services. An administrator can ask to increase the quota value of a certain service via ticket. This command helps administrators detect those issues in advance. From e014f13c186592cf3da37b4935f54cbd39203976 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Sun, 5 Jul 2020 18:40:59 +0100 Subject: [PATCH 4/8] New release --- cloudiscovery/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index b9ac67b..bb5884f 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -46,7 +46,7 @@ print("Python 3.6 or newer is required", file=sys.stderr) sys.exit(1) -__version__ = "2.1.1" +__version__ = "2.2.0" AVAILABLE_LANGUAGES = ["en_US", "pt_BR"] DEFAULT_REGION = "us-east-1" From 8c2e0cdb03f140c9dc409f2d344cd34492f0888c Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 00:18:47 +0100 Subject: [PATCH 5/8] Added more limit resources --- README.md | 4 ++ cloudiscovery/provider/limit/command.py | 60 ++++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 2 + 3 files changed, 66 insertions(+) diff --git a/README.md b/README.md index 64064eb..cacf6c1 100644 --- a/README.md +++ b/README.md @@ -295,16 +295,20 @@ With `--threshold 0-100` option, you can customize a minimum percentage threshol * appsync * autoscaling-plans * batch + * chime * codebuild * codecommit * codeguru reviewer + * codeguru profiler * cloudformation + * cloud map * dynamodb * ec2 * ecs * elasticfilesystem * elasticbeanstalk * elasticloadbalancing + * glue * iam * kms * mediaconnect diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index f7e7436..160566b 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -37,6 +37,10 @@ }, "global": False, }, + "AWSCloudMap": { + "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, + "global": False, + }, "batch": { "L-144F0CA5": { "method": "describe_compute_environments", @@ -45,6 +49,24 @@ }, "global": False, }, + "chime": { + "L-8EE806B4": { + "method": "list_voice_connectors", + "key": "VoiceConnectors", + "fields": [], + }, + "L-32405DBA": { + "method": "list_phone_numbers", + "key": "PhoneNumbers", + "fields": [], + }, + "L-D3615084": { + "method": "list_voice_connector_groups", + "key": "VoiceConnectorGroups", + "fields": [], + }, + "global": True, + }, "codebuild": { "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, "global": False, @@ -89,6 +111,7 @@ }, }, "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, + "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, "global": False, }, "codeguru-reviewer": { @@ -100,6 +123,14 @@ }, "global": False, }, + "codeguru-profiler": { + "L-DA8D4E8D": { + "method": "list_profiling_groups", + "key": "profilingGroupNames", + "fields": [], + }, + "global": False, + }, "dynamodb": { "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, "global": False, @@ -135,6 +166,11 @@ "key": "Applications", "fields": [], }, + "L-D64F1F14": { + "method": "describe_application_versions", + "key": "ApplicationVersions", + "fields": [], + }, "global": False, }, "elasticloadbalancing": { @@ -145,6 +181,29 @@ }, "global": False, }, + "glue": { + "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, + "L-D987EC31": { + "method": "get_user_defined_functions", + "key": "UserDefinedFunctions", + "fields": [], + "filter": {"Pattern": "*"}, + }, + "L-83192DBF": { + "method": "get_security_configurations", + "key": "SecurityConfigurations", + "fields": [], + }, + "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, + "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, + "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, + "L-04CEE988": { + "method": "list_ml_transforms", + "key": "TransformIds", + "fields": [], + }, + "global": False, + }, "iam": { "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, @@ -169,6 +228,7 @@ }, "kms": { "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, "global": False, }, "mediaconnect": { diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index 83c70cf..b713ce5 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -18,6 +18,8 @@ "elasticloadbalancing": "elbv2", "elasticfilesystem": "efs", "vpc": "ec2", + "codeguru-profiler": "codeguruprofiler", + "AWSCloudMap": "servicediscovery", } MAX_EXECUTION_PARALLEL = 3 From 909f8757e4bd3debf73e73bf04e161a9460cb47f Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 00:58:45 +0100 Subject: [PATCH 6/8] Added EC2 check --- cloudiscovery/provider/limit/command.py | 77 ++++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 5 +- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 160566b..bd32e00 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -141,6 +141,83 @@ "key": "Addresses", "fields": [], }, + "L-74FC7D96": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], + } + ] + }, + }, + "L-DB2E81BA": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "g3s.xlarge", + "g3.4xlarge", + "g3.8xlarge", + "g3.16xlarge", + "g4dn.xlarge", + "g4dn.2xlarge", + "g4dn.4xlarge", + "g4dn.8xlarge", + "g4dn.16xlarge", + "g4dn.12xlarge", + "g4dn.metal", + ], + } + ] + }, + }, + "L-1945791B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "inf1.xlarge", + "inf1.2xlarge", + "inf1.6xlarge", + "inf1.24xlarge", + ], + } + ] + }, + }, + "L-417A185B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "p2.xlarge", + "p2.8xlarge", + "p2.16xlarge", + "p3.2xlarge", + "p3.8xlarge", + "p3.16xlarge", + "p3dn.24xlarge", + ], + } + ] + }, + }, "global": False, }, "ecs": { diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index b713ce5..ee8fd54 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -148,7 +148,10 @@ def analyze_detail(self, client_quota, data_resource, service, threshold_request for page in pages: usage = usage + len(page[quota_data["key"]]) - percent = round((usage / value) * 100, 2) + try: + percent = round((usage / value) * 100, 2) + except ZeroDivisionError: + percent = 0 if percent >= threshold_requested: resources_found.append( From 36c5353a01b4b76c600222ee098064991f39e02a Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Mon, 6 Jul 2020 01:40:26 +0100 Subject: [PATCH 7/8] Added EC2 resources --- cloudiscovery/provider/limit/command.py | 419 +---------- .../provider/limit/data/allowed_resources.py | 651 ++++++++++++++++++ cloudiscovery/provider/limit/resource/all.py | 3 +- 3 files changed, 654 insertions(+), 419 deletions(-) create mode 100644 cloudiscovery/provider/limit/data/allowed_resources.py diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index bd32e00..9bd9693 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -3,424 +3,7 @@ from shared.common import ResourceCache, message_handler, Filterable, BaseOptions from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram - -ALLOWED_SERVICES_CODES = { - "acm": { - "L-F141DD1D": { - "method": "list_certificates", - "key": "CertificateSummaryList", - "fields": [], - }, - "global": False, - }, - "amplify": { - "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, - "global": False, - }, - "appmesh": { - "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, - "global": False, - }, - "appsync": { - "L-06A0647C": { - "method": "list_graphql_apis", - "key": "graphqlApis", - "fields": [], - }, - "global": False, - }, - "autoscaling-plans": { - "L-BD401546": { - "method": "describe_scaling_plans", - "key": "ScalingPlans", - "fields": [], - }, - "global": False, - }, - "AWSCloudMap": { - "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, - "global": False, - }, - "batch": { - "L-144F0CA5": { - "method": "describe_compute_environments", - "key": "computeEnvironments", - "fields": [], - }, - "global": False, - }, - "chime": { - "L-8EE806B4": { - "method": "list_voice_connectors", - "key": "VoiceConnectors", - "fields": [], - }, - "L-32405DBA": { - "method": "list_phone_numbers", - "key": "PhoneNumbers", - "fields": [], - }, - "L-D3615084": { - "method": "list_voice_connector_groups", - "key": "VoiceConnectorGroups", - "fields": [], - }, - "global": True, - }, - "codebuild": { - "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, - "global": False, - }, - "codecommit": { - "L-81790602": { - "method": "list_repositories", - "key": "repositories", - "fields": [], - }, - "global": False, - }, - "cloudformation": { - "L-0485CB21": { - "method": "list_stacks", - "key": "StackSummaries", - "fields": [], - "filter": { - "StackStatusFilter": [ - "CREATE_IN_PROGRESS", - "CREATE_FAILED", - "CREATE_COMPLETE", - "ROLLBACK_IN_PROGRESS", - "ROLLBACK_FAILED", - "ROLLBACK_COMPLETE", - "DELETE_IN_PROGRESS", - "DELETE_FAILED", - "UPDATE_IN_PROGRESS", - "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", - "UPDATE_COMPLETE", - "UPDATE_ROLLBACK_IN_PROGRESS", - "UPDATE_ROLLBACK_FAILED", - "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", - "UPDATE_ROLLBACK_COMPLETE", - "REVIEW_IN_PROGRESS", - "IMPORT_IN_PROGRESS", - "IMPORT_COMPLETE", - "IMPORT_ROLLBACK_IN_PROGRESS", - "IMPORT_ROLLBACK_FAILED", - "IMPORT_ROLLBACK_COMPLETE", - ] - }, - }, - "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, - "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, - "global": False, - }, - "codeguru-reviewer": { - "L-F5129FC6": { - "method": "list_code_reviews", - "key": "CodeReviewSummaries", - "fields": [], - "filter": {"Type": "PullRequest"}, - }, - "global": False, - }, - "codeguru-profiler": { - "L-DA8D4E8D": { - "method": "list_profiling_groups", - "key": "profilingGroupNames", - "fields": [], - }, - "global": False, - }, - "dynamodb": { - "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, - "global": False, - }, - "ec2": { - "L-0263D0A3": { - "method": "describe_addresses", - "key": "Addresses", - "fields": [], - }, - "L-74FC7D96": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], - } - ] - }, - }, - "L-DB2E81BA": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "g3s.xlarge", - "g3.4xlarge", - "g3.8xlarge", - "g3.16xlarge", - "g4dn.xlarge", - "g4dn.2xlarge", - "g4dn.4xlarge", - "g4dn.8xlarge", - "g4dn.16xlarge", - "g4dn.12xlarge", - "g4dn.metal", - ], - } - ] - }, - }, - "L-1945791B": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "inf1.xlarge", - "inf1.2xlarge", - "inf1.6xlarge", - "inf1.24xlarge", - ], - } - ] - }, - }, - "L-417A185B": { - "method": "describe_instances", - "key": "Reservations", - "fields": [], - "filter": { - "Filters": [ - { - "Name": "instance-type", - "Values": [ - "p2.xlarge", - "p2.8xlarge", - "p2.16xlarge", - "p3.2xlarge", - "p3.8xlarge", - "p3.16xlarge", - "p3dn.24xlarge", - ], - } - ] - }, - }, - "global": False, - }, - "ecs": { - "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, - "global": False, - }, - "elasticfilesystem": { - "L-848C634D": { - "method": "describe_file_systems", - "key": "FileSystems", - "fields": [], - }, - "global": False, - }, - "elasticbeanstalk": { - "L-8EFC1C51": { - "method": "describe_environments", - "key": "Environments", - "fields": [], - }, - "L-1CEABD17": { - "method": "describe_applications", - "key": "Applications", - "fields": [], - }, - "L-D64F1F14": { - "method": "describe_application_versions", - "key": "ApplicationVersions", - "fields": [], - }, - "global": False, - }, - "elasticloadbalancing": { - "L-53DA6B97": { - "method": "describe_load_balancers", - "key": "LoadBalancers", - "fields": [], - }, - "global": False, - }, - "glue": { - "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, - "L-D987EC31": { - "method": "get_user_defined_functions", - "key": "UserDefinedFunctions", - "fields": [], - "filter": {"Pattern": "*"}, - }, - "L-83192DBF": { - "method": "get_security_configurations", - "key": "SecurityConfigurations", - "fields": [], - }, - "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, - "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, - "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, - "L-04CEE988": { - "method": "list_ml_transforms", - "key": "TransformIds", - "fields": [], - }, - "global": False, - }, - "iam": { - "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, - "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, - "L-BF35879D": { - "method": "list_server_certificates", - "key": "ServerCertificateMetadataList", - "fields": [], - }, - "L-6E65F664": { - "method": "list_instance_profiles", - "key": "InstanceProfiles", - "fields": [], - "paginate": False, - }, - "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, - "L-DB618D39": { - "method": "list_saml_providers", - "key": "SAMLProviderList", - "fields": [], - }, - "global": True, - }, - "kms": { - "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, - "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, - "global": False, - }, - "mediaconnect": { - "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, - "L-F1F62F5D": { - "method": "list_entitlements", - "key": "Entitlements", - "fields": [], - }, - "global": False, - }, - "medialive": { - "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, - "L-BDF24E14": { - "method": "list_input_devices", - "key": "InputDevices", - "fields": [], - }, - "global": False, - }, - "mediapackage": { - "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, - "global": False, - }, - "qldb": { - "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, - "global": False, - }, - "robomaker": { - "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, - "L-D6554FB1": { - "method": "list_simulation_applications", - "key": "simulationApplicationSummaries", - "fields": [], - }, - "global": False, - }, - "route53": { - "L-4EA4796A": { - "method": "list_hosted_zones", - "key": "HostedZones", - "fields": [], - }, - "L-ACB674F3": { - "method": "list_health_checks", - "key": "HealthChecks", - "fields": [], - }, - "global": True, - }, - "route53resolver": { - "L-4A669CC0": { - "method": "list_resolver_endpoints", - "key": "ResolverEndpoints", - "fields": [], - }, - "L-51D8A1FB": { - "method": "list_resolver_rules", - "key": "ResolverRules", - "fields": [], - }, - "global": True, - }, - "rds": { - "L-7B6409FD": { - "method": "describe_db_instances", - "key": "DBInstances", - "fields": [], - }, - "L-952B80B8": { - "method": "describe_db_clusters", - "key": "DBClusters", - "fields": [], - }, - "L-DE55804A": { - "method": "describe_db_parameter_groups", - "key": "DBParameterGroups", - "fields": [], - }, - "L-9FA33840": { - "method": "describe_option_groups", - "key": "OptionGroupsList", - "fields": [], - }, - "global": False, - }, - "s3": { - "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, - "global": False, - }, - "sns": { - "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, - "global": False, - }, - "transcribe": { - "L-3278D334": { - "method": "list_vocabularies", - "key": "Vocabularies", - "fields": [], - }, - "global": False, - }, - "translate": { - "L-4011ABD8": { - "method": "list_terminologies", - "key": "TerminologyPropertiesList", - "fields": [], - }, - "global": False, - }, - "vpc": { - "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, - "global": False, - }, -} +from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES class LimitOptions(BaseAwsOptions, BaseOptions): diff --git a/cloudiscovery/provider/limit/data/allowed_resources.py b/cloudiscovery/provider/limit/data/allowed_resources.py new file mode 100644 index 0000000..25f1aa7 --- /dev/null +++ b/cloudiscovery/provider/limit/data/allowed_resources.py @@ -0,0 +1,651 @@ +ALLOWED_SERVICES_CODES = { + "acm": { + "L-F141DD1D": { + "method": "list_certificates", + "key": "CertificateSummaryList", + "fields": [], + }, + "global": False, + }, + "amplify": { + "L-1BED97F3": {"method": "list_apps", "key": "apps", "fields": [],}, + "global": False, + }, + "appmesh": { + "L-AC861A39": {"method": "list_meshes", "key": "meshes", "fields": [],}, + "global": False, + }, + "appsync": { + "L-06A0647C": { + "method": "list_graphql_apis", + "key": "graphqlApis", + "fields": [], + }, + "global": False, + }, + "autoscaling-plans": { + "L-BD401546": { + "method": "describe_scaling_plans", + "key": "ScalingPlans", + "fields": [], + }, + "global": False, + }, + "AWSCloudMap": { + "L-0FE3F50E": {"method": "list_namespaces", "key": "Namespaces", "fields": [],}, + "global": False, + }, + "batch": { + "L-144F0CA5": { + "method": "describe_compute_environments", + "key": "computeEnvironments", + "fields": [], + }, + "global": False, + }, + "chime": { + "L-8EE806B4": { + "method": "list_voice_connectors", + "key": "VoiceConnectors", + "fields": [], + }, + "L-32405DBA": { + "method": "list_phone_numbers", + "key": "PhoneNumbers", + "fields": [], + }, + "L-D3615084": { + "method": "list_voice_connector_groups", + "key": "VoiceConnectorGroups", + "fields": [], + }, + "global": True, + }, + "codebuild": { + "L-ACCF6C0D": {"method": "list_projects", "key": "projects", "fields": [],}, + "global": False, + }, + "codecommit": { + "L-81790602": { + "method": "list_repositories", + "key": "repositories", + "fields": [], + }, + "global": False, + }, + "cloudformation": { + "L-0485CB21": { + "method": "list_stacks", + "key": "StackSummaries", + "fields": [], + "filter": { + "StackStatusFilter": [ + "CREATE_IN_PROGRESS", + "CREATE_FAILED", + "CREATE_COMPLETE", + "ROLLBACK_IN_PROGRESS", + "ROLLBACK_FAILED", + "ROLLBACK_COMPLETE", + "DELETE_IN_PROGRESS", + "DELETE_FAILED", + "UPDATE_IN_PROGRESS", + "UPDATE_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_COMPLETE", + "UPDATE_ROLLBACK_IN_PROGRESS", + "UPDATE_ROLLBACK_FAILED", + "UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS", + "UPDATE_ROLLBACK_COMPLETE", + "REVIEW_IN_PROGRESS", + "IMPORT_IN_PROGRESS", + "IMPORT_COMPLETE", + "IMPORT_ROLLBACK_IN_PROGRESS", + "IMPORT_ROLLBACK_FAILED", + "IMPORT_ROLLBACK_COMPLETE", + ] + }, + }, + "L-9DE8E4FB": {"method": "list_types", "key": "TypeSummaries", "fields": [],}, + "L-31709F13": {"method": "list_stack_sets", "key": "Summaries", "fields": [],}, + "global": False, + }, + "codeguru-reviewer": { + "L-F5129FC6": { + "method": "list_code_reviews", + "key": "CodeReviewSummaries", + "fields": [], + "filter": {"Type": "PullRequest"}, + }, + "global": False, + }, + "codeguru-profiler": { + "L-DA8D4E8D": { + "method": "list_profiling_groups", + "key": "profilingGroupNames", + "fields": [], + }, + "global": False, + }, + "dynamodb": { + "L-F98FE922": {"method": "list_tables", "key": "TableNames", "fields": [],}, + "global": False, + }, + "ec2": { + "L-0263D0A3": { + "method": "describe_addresses", + "key": "Addresses", + "fields": [], + }, + "L-74FC7D96": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": ["f1.2xlarge", "f1.4xlarge", "f1.16xlarge"], + } + ] + }, + }, + "L-DB2E81BA": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "g3s.xlarge", + "g3.4xlarge", + "g3.8xlarge", + "g3.16xlarge", + "g4dn.xlarge", + "g4dn.2xlarge", + "g4dn.4xlarge", + "g4dn.8xlarge", + "g4dn.16xlarge", + "g4dn.12xlarge", + "g4dn.metal", + ], + } + ] + }, + }, + "L-1945791B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "inf1.xlarge", + "inf1.2xlarge", + "inf1.6xlarge", + "inf1.24xlarge", + ], + } + ] + }, + }, + "L-417A185B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "p2.xlarge", + "p2.8xlarge", + "p2.16xlarge", + "p3.2xlarge", + "p3.8xlarge", + "p3.16xlarge", + "p3dn.24xlarge", + ], + } + ] + }, + }, + "L-1216C47A": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "c5d.large", + "c5d.xlarge", + "c5d.2xlarge", + "c5d.4xlarge", + "c5d.9xlarge", + "c5d.12xlarge", + "c5d.18xlarge", + "c5d.24xlarge", + "c5d.metal", + "c5a.large", + "c5a.xlarge", + "c5a.2xlarge", + "c5a.4xlarge", + "c5a.8xlarge", + "c5a.12xlarge", + "c5a.16xlarge", + "c5a.24xlarge", + "c5n.large", + "c5n.xlarge", + "c5n.2xlarge", + "c5n.4xlarge", + "c5n.9xlarge", + "c5n.18xlarge", + "c5n.metal", + "c4.large", + "c4.xlarge", + "c4.2xlarge", + "c4.4xlarge", + "c4.8xlarge", + "d2.xlarge", + "d2.2xlarge", + "d2.4xlarge", + "d2.8xlarge", + "h1.2xlarge", + "h1.4xlarge", + "h1.8xlarge", + "h1.16xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", + "i3.metal", + "m6g.medium", + "m6g.large", + "m6g.xlarge", + "m6g.2xlarge", + "m6g.4xlarge", + "m6g.8xlarge", + "m6g.12xlarge", + "m6g.16xlarge", + "m6g.metal", + "m5.large", + "m5.xlarge", + "m5.2xlarge", + "m5.4xlarge", + "m5.8xlarge", + "m5.12xlarge", + "m5.16xlarge", + "m5.24xlarge", + "m5.metal", + "m5d.large", + "m5d.xlarge", + "m5d.2xlarge", + "m5d.4xlarge", + "m5d.8xlarge", + "m5d.12xlarge", + "m5d.16xlarge", + "m5d.24xlarge", + "m5d.metal", + "m5a.large", + "m5a.xlarge", + "m5a.2xlarge", + "m5a.4xlarge", + "m5a.8xlarge", + "m5a.12xlarge", + "m5a.16xlarge", + "m5a.24xlarge", + "m5ad.large", + "m5ad.xlarge", + "m5ad.2xlarge", + "m5ad.4xlarge", + "m5ad.12xlarge", + "m5ad.24xlarge", + "m5n.large", + "m5n.xlarge", + "m5n.2xlarge", + "m5n.4xlarge", + "m5n.8xlarge", + "m5n.12xlarge", + "m5n.16xlarge", + "m5n.24xlarge", + "m5dn.large", + "m5dn.xlarge", + "m5dn.2xlarge", + "m5dn.4xlarge", + "m5dn.8xlarge", + "m5dn.12xlarge", + "m5dn.16xlarge", + "m5dn.24xlarge", + "m4.large", + "m4.xlarge", + "m4.2xlarge", + "m4.4xlarge", + "m4.10xlarge", + "m4.16xlarge", + "z1d.large", + "z1d.xlarge", + "z1d.2xlarge", + "z1d.3xlarge", + "z1d.6xlarge", + "z1d.12xlarge", + "z1d.metal", + "r6g.medium", + "r6g.large", + "r6g.xlarge", + "r6g.2xlarge", + "r6g.4xlarge", + "r6g.8xlarge", + "r6g.12xlarge", + "r6g.16xlarge", + "r6g.metal", + "r5.large", + "r5.xlarge", + "r5.2xlarge", + "r5.4xlarge", + "r5.8xlarge", + "r5.12xlarge", + "r5.16xlarge", + "r5.24xlarge", + "r5.metal", + "r5d.large", + "r5d.xlarge", + "r5d.2xlarge", + "r5d.4xlarge", + "r5d.8xlarge", + "r5d.12xlarge", + "r5d.16xlarge", + "r5d.24xlarge", + "r5d.metal", + "r5a.large", + "r5a.xlarge", + "r5a.2xlarge", + "r5a.4xlarge", + "r5a.8xlarge", + "r5a.12xlarge", + "r5a.16xlarge", + "r5a.24xlarge", + "r5ad.large", + "r5ad.xlarge", + "r5ad.2xlarge", + "r5ad.4xlarge", + "r5ad.12xlarge", + "r5ad.24xlarge", + "r5n.large", + "r5n.xlarge", + "r5n.2xlarge", + "r5n.4xlarge", + "r5n.8xlarge", + "r5n.12xlarge", + "r5n.16xlarge", + "r5n.24xlarge", + "r5dn.large", + "r5dn.xlarge", + "r5dn.2xlarge", + "r5dn.4xlarge", + "r5dn.8xlarge", + "r5dn.12xlarge", + "r5dn.16xlarge", + "r5dn.24xlarge", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", + "t3.nano", + "t3.micro", + "t3.small", + "t3.medium", + "t3.large", + "t3.xlarge", + "t3.2xlarge", + "t3a.nano", + "t3a.micro", + "t3a.small", + "t3a.medium", + "t3a.large", + "t3a.xlarge", + "t3a.2xlarge", + "t2.nano", + "t2.micro", + "t2.small", + "t2.medium", + "t2.large", + "t2.xlarge", + "t2.2xlarge", + ], + } + ] + }, + }, + "L-7295265B": { + "method": "describe_instances", + "key": "Reservations", + "fields": [], + "filter": { + "Filters": [ + { + "Name": "instance-type", + "Values": [ + "x1e.xlarge", + "x1e.2xlarge", + "x1e.4xlarge", + "x1e.8xlarge", + "x1e.16xlarge", + "x1e.32xlarge", + "x1.16xlarge", + "x1.32xlarge", + ], + } + ] + }, + }, + "global": False, + }, + "ecs": { + "L-21C621EB": {"method": "list_clusters", "key": "clusterArns", "fields": [],}, + "global": False, + }, + "elasticfilesystem": { + "L-848C634D": { + "method": "describe_file_systems", + "key": "FileSystems", + "fields": [], + }, + "global": False, + }, + "elasticbeanstalk": { + "L-8EFC1C51": { + "method": "describe_environments", + "key": "Environments", + "fields": [], + }, + "L-1CEABD17": { + "method": "describe_applications", + "key": "Applications", + "fields": [], + }, + "L-D64F1F14": { + "method": "describe_application_versions", + "key": "ApplicationVersions", + "fields": [], + }, + "global": False, + }, + "elasticloadbalancing": { + "L-53DA6B97": { + "method": "describe_load_balancers", + "key": "LoadBalancers", + "fields": [], + }, + "global": False, + }, + "glue": { + "L-F953935E": {"method": "get_databases", "key": "DatabaseList", "fields": [],}, + "L-D987EC31": { + "method": "get_user_defined_functions", + "key": "UserDefinedFunctions", + "fields": [], + "filter": {"Pattern": "*"}, + }, + "L-83192DBF": { + "method": "get_security_configurations", + "key": "SecurityConfigurations", + "fields": [], + }, + "L-F1653A6D": {"method": "get_triggers", "key": "Triggers", "fields": [],}, + "L-11FA2C1A": {"method": "get_crawlers", "key": "Crawlers", "fields": [],}, + "L-7DD7C33A": {"method": "list_workflows", "key": "Workflows", "fields": [],}, + "L-04CEE988": { + "method": "list_ml_transforms", + "key": "TransformIds", + "fields": [], + }, + "global": False, + }, + "iam": { + "L-F4A5425F": {"method": "list_groups", "key": "Groups", "fields": [],}, + "L-F55AF5E4": {"method": "list_users", "key": "Users", "fields": [],}, + "L-BF35879D": { + "method": "list_server_certificates", + "key": "ServerCertificateMetadataList", + "fields": [], + }, + "L-6E65F664": { + "method": "list_instance_profiles", + "key": "InstanceProfiles", + "fields": [], + "paginate": False, + }, + "L-FE177D64": {"method": "list_roles", "key": "Roles", "fields": [],}, + "L-DB618D39": { + "method": "list_saml_providers", + "key": "SAMLProviderList", + "fields": [], + }, + "global": True, + }, + "kms": { + "L-C2F1777E": {"method": "list_keys", "key": "Keys", "fields": [],}, + "L-2601EE20": {"method": "list_aliases", "key": "Aliases", "fields": [],}, + "global": False, + }, + "mediaconnect": { + "L-A99016A8": {"method": "list_flows", "key": "Flows", "fields": [],}, + "L-F1F62F5D": { + "method": "list_entitlements", + "key": "Entitlements", + "fields": [], + }, + "global": False, + }, + "medialive": { + "L-D1AFAF75": {"method": "list_channels", "key": "Channels", "fields": [],}, + "L-BDF24E14": { + "method": "list_input_devices", + "key": "InputDevices", + "fields": [], + }, + "global": False, + }, + "mediapackage": { + "L-352B8598": {"method": "list_channels", "key": "Channels", "fields": [],}, + "global": False, + }, + "qldb": { + "L-CD70CADB": {"method": "list_ledgers", "key": "Ledgers", "fields": [],}, + "global": False, + }, + "robomaker": { + "L-40FACCBF": {"method": "list_robots", "key": "robots", "fields": [],}, + "L-D6554FB1": { + "method": "list_simulation_applications", + "key": "simulationApplicationSummaries", + "fields": [], + }, + "global": False, + }, + "route53": { + "L-4EA4796A": { + "method": "list_hosted_zones", + "key": "HostedZones", + "fields": [], + }, + "L-ACB674F3": { + "method": "list_health_checks", + "key": "HealthChecks", + "fields": [], + }, + "global": True, + }, + "route53resolver": { + "L-4A669CC0": { + "method": "list_resolver_endpoints", + "key": "ResolverEndpoints", + "fields": [], + }, + "L-51D8A1FB": { + "method": "list_resolver_rules", + "key": "ResolverRules", + "fields": [], + }, + "global": True, + }, + "rds": { + "L-7B6409FD": { + "method": "describe_db_instances", + "key": "DBInstances", + "fields": [], + }, + "L-952B80B8": { + "method": "describe_db_clusters", + "key": "DBClusters", + "fields": [], + }, + "L-DE55804A": { + "method": "describe_db_parameter_groups", + "key": "DBParameterGroups", + "fields": [], + }, + "L-9FA33840": { + "method": "describe_option_groups", + "key": "OptionGroupsList", + "fields": [], + }, + "global": False, + }, + "s3": { + "L-DC2B2D3D": {"method": "list_buckets", "key": "Buckets", "fields": [],}, + "global": False, + }, + "sns": { + "L-61103206": {"method": "list_topics", "key": "Topics", "fields": [],}, + "global": False, + }, + "transcribe": { + "L-3278D334": { + "method": "list_vocabularies", + "key": "Vocabularies", + "fields": [], + }, + "global": False, + }, + "translate": { + "L-4011ABD8": { + "method": "list_terminologies", + "key": "TerminologyPropertiesList", + "fields": [], + }, + "global": False, + }, + "vpc": { + "L-F678F1CE": {"method": "describe_vpcs", "key": "Vpcs", "fields": [],}, + "global": False, + }, +} diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index ee8fd54..b73c299 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -2,7 +2,8 @@ from concurrent.futures.thread import ThreadPoolExecutor -from provider.limit.command import LimitOptions, ALLOWED_SERVICES_CODES +from provider.limit.command import LimitOptions +from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES from shared.common import ( ResourceProvider, Resource, From c4041a40f65008072c22aa4f00ee1db4a29afd39 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Mon, 6 Jul 2020 14:10:34 +0700 Subject: [PATCH 8/8] limits - code review --- README.md | 16 +- cloudiscovery/provider/limit/command.py | 67 +++++-- cloudiscovery/provider/limit/resource/all.py | 192 ++++++++++--------- cloudiscovery/shared/error_handler.py | 8 +- 4 files changed, 163 insertions(+), 120 deletions(-) diff --git a/README.md b/README.md index cacf6c1..1e5f3f2 100644 --- a/README.md +++ b/README.md @@ -150,9 +150,19 @@ The configured credentials must be associated to a user or role with proper perm "amplify:ListApps", "autoscaling-plans:DescribeScalingPlans", "medialive:ListChannels", + "medialive:ListInputDevices", "mediapackage:ListChannels", "qldb:ListLedgers", - "transcribe:ListVocabularies" + "transcribe:ListVocabularies", + "glue:GetDatabases", + "glue:GetUserDefinedFunctions", + "glue:GetSecurityConfigurations", + "glue:GetTriggers", + "glue:GetCrawlers", + "glue:ListWorkflows", + "glue:ListMLTransforms", + "codeguru-reviewer:ListCodeReviews", + "servicediscovery:ListNamespaces" ], "Resource": [ "*" ] } @@ -282,9 +292,9 @@ Types of resources mostly cover Terraform types. It is possible to narrow down s ### AWS Limit -It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource. +It's possible to check resources limits across various service in an account. This command implements over 60 limits checks. -With `--services value,value,value` selection, you can narrow down checks to services that you want to check. +With `--services value,value,value` parameter, you can narrow down checks to just services that you want to check. With `--threshold 0-100` option, you can customize a minimum percentage threshold to start reporting a warning. diff --git a/cloudiscovery/provider/limit/command.py b/cloudiscovery/provider/limit/command.py index 9bd9693..83f9bd1 100644 --- a/cloudiscovery/provider/limit/command.py +++ b/cloudiscovery/provider/limit/command.py @@ -1,6 +1,12 @@ from typing import List -from shared.common import ResourceCache, message_handler, Filterable, BaseOptions +from shared.common import ( + ResourceCache, + message_handler, + Filterable, + BaseOptions, + log_critical, +) from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner from shared.diagram import NoDiagram from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES @@ -27,10 +33,11 @@ def __init__( class LimitParameters: - def __init__(self, session, region: str, services): + def __init__(self, session, region: str, services, options: LimitOptions): self.region = region self.cache = ResourceCache() self.session = session + self.options = options self.services = [] if services is None: for service in ALLOWED_SERVICES_CODES: @@ -52,12 +59,13 @@ def init_globalaws_limits_cache(self): if cache is not None: continue - message_handler( - "Fetching aws global limit to service {} in region {} to cache...".format( - service_code, self.region - ), - "HEADER", - ) + if self.options.verbose: + message_handler( + "Fetching aws global limit to service {} in region {} to cache...".format( + service_code, self.region + ), + "HEADER", + ) cache_codes = dict() for quota_code in ALLOWED_SERVICES_CODES[service_code]: @@ -76,16 +84,11 @@ def init_globalaws_limits_cache(self): "service-quotas", region_name=self.region ) - response = service_quota.get_aws_default_service_quota( - ServiceCode=service_code, QuotaCode=quota_code + item_to_add = self.get_quota( + quota_code, service_code, service_quota ) - - item_to_add = { - "value": response["Quota"]["Value"], - "adjustable": response["Quota"]["Adjustable"], - "quota_code": quota_code, - "quota_name": response["Quota"]["QuotaName"], - } + if item_to_add is None: + continue if service_code in cache_codes: cache_codes[service_code].append(item_to_add) @@ -96,6 +99,28 @@ def init_globalaws_limits_cache(self): return True + def get_quota(self, quota_code, service_code, service_quota): + try: + response = service_quota.get_aws_default_service_quota( + ServiceCode=service_code, QuotaCode=quota_code + ) + # pylint: disable=broad-except + except Exception as e: + if self.options.verbose: + log_critical( + "\nCannot take quota {} for {}: {}".format( + quota_code, service_code, str(e) + ) + ) + return None + item_to_add = { + "value": response["Quota"]["Value"], + "adjustable": response["Quota"]["Adjustable"], + "quota_code": quota_code, + "quota_name": response["Quota"]["QuotaName"], + } + return item_to_add + class Limit(BaseAwsCommand): def __init__(self, region_names, session, threshold): @@ -109,10 +134,10 @@ def __init__(self, region_names, session, threshold): super().__init__(region_names, session) self.threshold = threshold - def init_globalaws_limits_cache(self, region, services): + def init_globalaws_limits_cache(self, region, services, options: LimitOptions): # Cache services global and local services LimitParameters( - session=self.session, region=region, services=services + session=self.session, region=region, services=services, options=options ).init_globalaws_limits_cache() def run( @@ -128,7 +153,6 @@ def run( services.append(service) for region in self.region_names: - self.init_globalaws_limits_cache(region=region, services=services) limit_options = LimitOptions( verbose=verbose, filters=filters, @@ -137,6 +161,9 @@ def run( services=services, threshold=self.threshold, ) + self.init_globalaws_limits_cache( + region=region, services=services, options=limit_options + ) command_runner = AwsCommandRunner(services=services) command_runner.run( diff --git a/cloudiscovery/provider/limit/resource/all.py b/cloudiscovery/provider/limit/resource/all.py index b73c299..e2690d0 100644 --- a/cloudiscovery/provider/limit/resource/all.py +++ b/cloudiscovery/provider/limit/resource/all.py @@ -23,7 +23,7 @@ "AWSCloudMap": "servicediscovery", } -MAX_EXECUTION_PARALLEL = 3 +MAX_EXECUTION_PARALLEL = 2 class LimitResources(ResourceProvider): @@ -53,8 +53,8 @@ def get_resources(self) -> List[Resource]: with ThreadPoolExecutor(MAX_EXECUTION_PARALLEL) as executor: results = executor.map( - lambda aws_limit: self.analyze_service( - aws_limit=aws_limit, + lambda service_name: self.analyze_service( + service_name=service_name, client_quota=client_quota, threshold_requested=int(threshold_requested), ), @@ -68,110 +68,112 @@ def get_resources(self) -> List[Resource]: return resources_found @exception - def analyze_service(self, aws_limit, client_quota, threshold_requested): - - service = aws_limit - - cache_key = "aws_limits_" + service + "_" + self.options.region_name + def analyze_service(self, service_name, client_quota, threshold_requested): + cache_key = "aws_limits_" + service_name + "_" + self.options.region_name cache = self.cache.get_key(cache_key) - - return self.analyze_detail( - client_quota=client_quota, - data_resource=cache[service], - service=service, - threshold_requested=threshold_requested, - ) + resources_found = [] + if service_name not in cache: + return [] + + for data_quota_code in cache[service_name]: + if data_quota_code is None: + continue + resource_found = self.analyze_quota( + client_quota=client_quota, + data_quota_code=data_quota_code, + service=service_name, + threshold_requested=threshold_requested, + ) + if resource_found is not None: + resources_found.append(resource_found) + return resources_found @exception # pylint: disable=too-many-locals - def analyze_detail(self, client_quota, data_resource, service, threshold_requested): + def analyze_quota( + self, client_quota, data_quota_code, service, threshold_requested + ): + resource_found = None + quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] - resources_found = [] - - for data_quota_code in data_resource: - - quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]] - - value_aws = value = data_quota_code["value"] + value_aws = value = data_quota_code["value"] - # Quota is adjustable by ticket request, then must override this values. - if bool(data_quota_code["adjustable"]) is True: - try: - response_quota = client_quota.get_service_quota( - ServiceCode=service, QuotaCode=data_quota_code["quota_code"] - ) - if "Value" in response_quota["Quota"]: - value = response_quota["Quota"]["Value"] - else: - value = data_quota_code["value"] - except client_quota.exceptions.NoSuchResourceException: + # Quota is adjustable by ticket request, then must override this values. + if bool(data_quota_code["adjustable"]) is True: + try: + response_quota = client_quota.get_service_quota( + ServiceCode=service, QuotaCode=data_quota_code["quota_code"] + ) + if "Value" in response_quota["Quota"]: + value = response_quota["Quota"]["Value"] + else: value = data_quota_code["value"] + except client_quota.exceptions.NoSuchResourceException: + value = data_quota_code["value"] + + if self.options.verbose: + message_handler( + "Collecting data from Quota: " + + service + + " - " + + data_quota_code["quota_name"] + + "...", + "HEADER", + ) - if self.options.verbose: - message_handler( - "Collecting data from Quota: " - + service - + " - " - + data_quota_code["quota_name"] - + "...", - "HEADER", - ) + # Need to convert some quota-services endpoint + if service in SERVICEQUOTA_TO_BOTO3: + service = SERVICEQUOTA_TO_BOTO3.get(service) - # Need to convert some quota-services endpoint - if service in SERVICEQUOTA_TO_BOTO3: - service = SERVICEQUOTA_TO_BOTO3.get(service) + client = self.options.session.client( + service, region_name=self.options.region_name + ) - client = self.options.session.client( - service, region_name=self.options.region_name - ) + usage = 0 - usage = 0 + # Check filters by resource + if "filter" in quota_data: + filters = quota_data["filter"] + else: + filters = None - # Check filters by resource - if "filter" in quota_data: - filters = quota_data["filter"] + pages = get_paginator( + client=client, + operation_name=quota_data["method"], + resource_type="aws_limit", + filters=filters, + ) + if not pages: + if filters: + response = getattr(client, quota_data["method"])(**filters) else: - filters = None - - pages = get_paginator( - client=client, - operation_name=quota_data["method"], - resource_type="aws_limit", - filters=filters, + response = getattr(client, quota_data["method"])() + usage = len(response[quota_data["key"]]) + else: + for page in pages: + usage = usage + len(page[quota_data["key"]]) + + try: + percent = round((usage / value) * 100, 2) + except ZeroDivisionError: + percent = 0 + + if percent >= threshold_requested: + resource_found = Resource( + digest=ResourceDigest( + id=data_quota_code["quota_code"], type="aws_limit" + ), + name="", + group="", + limits=LimitsValues( + quota_name=data_quota_code["quota_name"], + quota_code=data_quota_code["quota_code"], + aws_limit=int(value_aws), + local_limit=int(value), + usage=int(usage), + service=service, + percent=percent, + ), ) - if not pages: - if filters: - response = getattr(client, quota_data["method"])(**filters) - else: - response = getattr(client, quota_data["method"])() - usage = len(response[quota_data["key"]]) - else: - for page in pages: - usage = usage + len(page[quota_data["key"]]) - try: - percent = round((usage / value) * 100, 2) - except ZeroDivisionError: - percent = 0 - - if percent >= threshold_requested: - resources_found.append( - Resource( - digest=ResourceDigest( - id=data_quota_code["quota_code"], type="aws_limit" - ), - name="", - group="", - limits=LimitsValues( - quota_name=data_quota_code["quota_name"], - quota_code=data_quota_code["quota_code"], - aws_limit=int(value_aws), - local_limit=int(value), - usage=int(usage), - service=service, - percent=percent, - ), - ) - ) - - return resources_found + return resource_found diff --git a/cloudiscovery/shared/error_handler.py b/cloudiscovery/shared/error_handler.py index 0be51cd..835182f 100644 --- a/cloudiscovery/shared/error_handler.py +++ b/cloudiscovery/shared/error_handler.py @@ -11,13 +11,17 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) # pylint: disable=broad-except except Exception as e: - if "Could not connect to the endpoint URL" in str(e): + exception_str = str(e) + if ( + "Could not connect to the endpoint URL" in exception_str + or "the specified service does not exist" in exception_str + ): message = "\nThe service {} is not available in this region".format( func.__qualname__ ) else: message = "\nError running check {}. Error message {}".format( - func.__qualname__, str(e) + func.__qualname__, exception_str ) log_critical(message)