Skip to content

Commit

Permalink
limits - code review
Browse files Browse the repository at this point in the history
  • Loading branch information
@meshuga
meshuga committed Jul 6, 2020
1 parent 36c5353 commit c4041a4
Show file tree
Hide file tree
Showing 4 changed files with 163 additions and 120 deletions.
16 changes: 13 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,19 @@ The configured credentials must be associated to a user or role with proper perm
"amplify:ListApps",
"autoscaling-plans:DescribeScalingPlans",
"medialive:ListChannels",
"medialive:ListInputDevices",
"mediapackage:ListChannels",
"qldb:ListLedgers",
"transcribe:ListVocabularies"
"transcribe:ListVocabularies",
"glue:GetDatabases",
"glue:GetUserDefinedFunctions",
"glue:GetSecurityConfigurations",
"glue:GetTriggers",
"glue:GetCrawlers",
"glue:ListWorkflows",
"glue:ListMLTransforms",
"codeguru-reviewer:ListCodeReviews",
"servicediscovery:ListNamespaces"
],
"Resource": [ "*" ]
}
Expand Down Expand Up @@ -282,9 +292,9 @@ Types of resources mostly cover Terraform types. It is possible to narrow down s
### AWS Limit
It's possible to check resources limits in an account. This script allows check all available services or check only a specific resource.
It's possible to check resources limits across various service in an account. This command implements over 60 limits checks.
With `--services value,value,value` selection, you can narrow down checks to services that you want to check.
With `--services value,value,value` parameter, you can narrow down checks to just services that you want to check.
With `--threshold 0-100` option, you can customize a minimum percentage threshold to start reporting a warning.
Expand Down
67 changes: 47 additions & 20 deletions cloudiscovery/provider/limit/command.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
from typing import List

from shared.common import ResourceCache, message_handler, Filterable, BaseOptions
from shared.common import (
ResourceCache,
message_handler,
Filterable,
BaseOptions,
log_critical,
)
from shared.common_aws import BaseAwsOptions, BaseAwsCommand, AwsCommandRunner
from shared.diagram import NoDiagram
from provider.limit.data.allowed_resources import ALLOWED_SERVICES_CODES
Expand All @@ -27,10 +33,11 @@ def __init__(


class LimitParameters:
def __init__(self, session, region: str, services):
def __init__(self, session, region: str, services, options: LimitOptions):
self.region = region
self.cache = ResourceCache()
self.session = session
self.options = options
self.services = []
if services is None:
for service in ALLOWED_SERVICES_CODES:
Expand All @@ -52,12 +59,13 @@ def init_globalaws_limits_cache(self):
if cache is not None:
continue

message_handler(
"Fetching aws global limit to service {} in region {} to cache...".format(
service_code, self.region
),
"HEADER",
)
if self.options.verbose:
message_handler(
"Fetching aws global limit to service {} in region {} to cache...".format(
service_code, self.region
),
"HEADER",
)

cache_codes = dict()
for quota_code in ALLOWED_SERVICES_CODES[service_code]:
Expand All @@ -76,16 +84,11 @@ def init_globalaws_limits_cache(self):
"service-quotas", region_name=self.region
)

response = service_quota.get_aws_default_service_quota(
ServiceCode=service_code, QuotaCode=quota_code
item_to_add = self.get_quota(
quota_code, service_code, service_quota
)

item_to_add = {
"value": response["Quota"]["Value"],
"adjustable": response["Quota"]["Adjustable"],
"quota_code": quota_code,
"quota_name": response["Quota"]["QuotaName"],
}
if item_to_add is None:
continue

if service_code in cache_codes:
cache_codes[service_code].append(item_to_add)
Expand All @@ -96,6 +99,28 @@ def init_globalaws_limits_cache(self):

return True

def get_quota(self, quota_code, service_code, service_quota):
try:
response = service_quota.get_aws_default_service_quota(
ServiceCode=service_code, QuotaCode=quota_code
)
# pylint: disable=broad-except
except Exception as e:
if self.options.verbose:
log_critical(
"\nCannot take quota {} for {}: {}".format(
quota_code, service_code, str(e)
)
)
return None
item_to_add = {
"value": response["Quota"]["Value"],
"adjustable": response["Quota"]["Adjustable"],
"quota_code": quota_code,
"quota_name": response["Quota"]["QuotaName"],
}
return item_to_add


class Limit(BaseAwsCommand):
def __init__(self, region_names, session, threshold):
Expand All @@ -109,10 +134,10 @@ def __init__(self, region_names, session, threshold):
super().__init__(region_names, session)
self.threshold = threshold

def init_globalaws_limits_cache(self, region, services):
def init_globalaws_limits_cache(self, region, services, options: LimitOptions):
# Cache services global and local services
LimitParameters(
session=self.session, region=region, services=services
session=self.session, region=region, services=services, options=options
).init_globalaws_limits_cache()

def run(
Expand All @@ -128,7 +153,6 @@ def run(
services.append(service)

for region in self.region_names:
self.init_globalaws_limits_cache(region=region, services=services)
limit_options = LimitOptions(
verbose=verbose,
filters=filters,
Expand All @@ -137,6 +161,9 @@ def run(
services=services,
threshold=self.threshold,
)
self.init_globalaws_limits_cache(
region=region, services=services, options=limit_options
)

command_runner = AwsCommandRunner(services=services)
command_runner.run(
Expand Down
192 changes: 97 additions & 95 deletions cloudiscovery/provider/limit/resource/all.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"AWSCloudMap": "servicediscovery",
}

MAX_EXECUTION_PARALLEL = 3
MAX_EXECUTION_PARALLEL = 2


class LimitResources(ResourceProvider):
Expand Down Expand Up @@ -53,8 +53,8 @@ def get_resources(self) -> List[Resource]:

with ThreadPoolExecutor(MAX_EXECUTION_PARALLEL) as executor:
results = executor.map(
lambda aws_limit: self.analyze_service(
aws_limit=aws_limit,
lambda service_name: self.analyze_service(
service_name=service_name,
client_quota=client_quota,
threshold_requested=int(threshold_requested),
),
Expand All @@ -68,110 +68,112 @@ def get_resources(self) -> List[Resource]:
return resources_found

@exception
def analyze_service(self, aws_limit, client_quota, threshold_requested):

service = aws_limit

cache_key = "aws_limits_" + service + "_" + self.options.region_name
def analyze_service(self, service_name, client_quota, threshold_requested):
cache_key = "aws_limits_" + service_name + "_" + self.options.region_name
cache = self.cache.get_key(cache_key)

return self.analyze_detail(
client_quota=client_quota,
data_resource=cache[service],
service=service,
threshold_requested=threshold_requested,
)
resources_found = []
if service_name not in cache:
return []

for data_quota_code in cache[service_name]:
if data_quota_code is None:
continue
resource_found = self.analyze_quota(
client_quota=client_quota,
data_quota_code=data_quota_code,
service=service_name,
threshold_requested=threshold_requested,
)
if resource_found is not None:
resources_found.append(resource_found)
return resources_found

@exception
# pylint: disable=too-many-locals
def analyze_detail(self, client_quota, data_resource, service, threshold_requested):
def analyze_quota(
self, client_quota, data_quota_code, service, threshold_requested
):
resource_found = None
quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]]

resources_found = []

for data_quota_code in data_resource:

quota_data = ALLOWED_SERVICES_CODES[service][data_quota_code["quota_code"]]

value_aws = value = data_quota_code["value"]
value_aws = value = data_quota_code["value"]

# Quota is adjustable by ticket request, then must override this values.
if bool(data_quota_code["adjustable"]) is True:
try:
response_quota = client_quota.get_service_quota(
ServiceCode=service, QuotaCode=data_quota_code["quota_code"]
)
if "Value" in response_quota["Quota"]:
value = response_quota["Quota"]["Value"]
else:
value = data_quota_code["value"]
except client_quota.exceptions.NoSuchResourceException:
# Quota is adjustable by ticket request, then must override this values.
if bool(data_quota_code["adjustable"]) is True:
try:
response_quota = client_quota.get_service_quota(
ServiceCode=service, QuotaCode=data_quota_code["quota_code"]
)
if "Value" in response_quota["Quota"]:
value = response_quota["Quota"]["Value"]
else:
value = data_quota_code["value"]
except client_quota.exceptions.NoSuchResourceException:
value = data_quota_code["value"]

if self.options.verbose:
message_handler(
"Collecting data from Quota: "
+ service
+ " - "
+ data_quota_code["quota_name"]
+ "...",
"HEADER",
)

if self.options.verbose:
message_handler(
"Collecting data from Quota: "
+ service
+ " - "
+ data_quota_code["quota_name"]
+ "...",
"HEADER",
)
# Need to convert some quota-services endpoint
if service in SERVICEQUOTA_TO_BOTO3:
service = SERVICEQUOTA_TO_BOTO3.get(service)

# Need to convert some quota-services endpoint
if service in SERVICEQUOTA_TO_BOTO3:
service = SERVICEQUOTA_TO_BOTO3.get(service)
client = self.options.session.client(
service, region_name=self.options.region_name
)

client = self.options.session.client(
service, region_name=self.options.region_name
)
usage = 0

usage = 0
# Check filters by resource
if "filter" in quota_data:
filters = quota_data["filter"]
else:
filters = None

# Check filters by resource
if "filter" in quota_data:
filters = quota_data["filter"]
pages = get_paginator(
client=client,
operation_name=quota_data["method"],
resource_type="aws_limit",
filters=filters,
)
if not pages:
if filters:
response = getattr(client, quota_data["method"])(**filters)
else:
filters = None

pages = get_paginator(
client=client,
operation_name=quota_data["method"],
resource_type="aws_limit",
filters=filters,
response = getattr(client, quota_data["method"])()
usage = len(response[quota_data["key"]])
else:
for page in pages:
usage = usage + len(page[quota_data["key"]])

try:
percent = round((usage / value) * 100, 2)
except ZeroDivisionError:
percent = 0

if percent >= threshold_requested:
resource_found = Resource(
digest=ResourceDigest(
id=data_quota_code["quota_code"], type="aws_limit"
),
name="",
group="",
limits=LimitsValues(
quota_name=data_quota_code["quota_name"],
quota_code=data_quota_code["quota_code"],
aws_limit=int(value_aws),
local_limit=int(value),
usage=int(usage),
service=service,
percent=percent,
),
)
if not pages:
if filters:
response = getattr(client, quota_data["method"])(**filters)
else:
response = getattr(client, quota_data["method"])()
usage = len(response[quota_data["key"]])
else:
for page in pages:
usage = usage + len(page[quota_data["key"]])

try:
percent = round((usage / value) * 100, 2)
except ZeroDivisionError:
percent = 0

if percent >= threshold_requested:
resources_found.append(
Resource(
digest=ResourceDigest(
id=data_quota_code["quota_code"], type="aws_limit"
),
name="",
group="",
limits=LimitsValues(
quota_name=data_quota_code["quota_name"],
quota_code=data_quota_code["quota_code"],
aws_limit=int(value_aws),
local_limit=int(value),
usage=int(usage),
service=service,
percent=percent,
),
)
)

return resources_found
return resource_found
Loading

0 comments on commit c4041a4

Please sign in to comment.