diff --git a/.coveragerc b/.coveragerc index f917935c..e64baf09 100644 --- a/.coveragerc +++ b/.coveragerc @@ -13,3 +13,4 @@ exclude_lines = raise NotImplementedError except ImportError: pragma: no cover + .*# nocoverage.* diff --git a/.landscape.yaml b/.landscape.yaml deleted file mode 100644 index d13c797c..00000000 --- a/.landscape.yaml +++ /dev/null @@ -1,8 +0,0 @@ -doc-warnings: true -strictness: medium -test-warnings: false -pylint: - disable: - - protected-access -ignore-paths: - - awslimitchecker/services/__init__.py diff --git a/.travis.yml b/.travis.yml index c4bd7405..2120c49c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,17 +15,21 @@ matrix: env: TOXENV=py37 dist: xenial sudo: true + - python: "3.8" + env: TOXENV=py38 + dist: xenial + sudo: true - python: "3.7" env: TOXENV=docker dist: xenial sudo: true services: - docker - - python: "2.7" + - python: "3.7" env: TOXENV=docs - python: "2.7" env: TOXENV=integration - - python: "3.6" + - python: "3.7" env: TOXENV=integration3 install: - git config --global user.email "travisci@jasonantman.com" diff --git a/CHANGES.rst b/CHANGES.rst index 111bbf72..0231e11d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,72 @@ Changelog ========= +.. _changelog.8_0_0: + +8.0.0 (2019-11-03) +------------------ + +**Important:** This release includes **major** changes to the EC2 On-Demand Instances service limits! For most users, this means the 175 Instance-type-specific limits will be removed and replaced with five (5) limits. Please see the :ref:`changelog.8_0_0_vcpu_limits` section below for further details, as this will especially impact anyone using limit or threshold overrides, or post-processing awslimitchecker's output. This is also a time to remind all users that this project adheres to a strict :ref:`development.versioning_policy` and if occasional breakage due to limit or IAM policy changes is unacceptable, you should pin to a major version. + +**Important:** Python versions prior to 3.5, including 2.7, are now pending deprecation. As of January 1, 2020, they will no longer be tested or supported, and awslimitchecker **will require Python 3.5 or newer**. Please see below for details. Also take note that running via the official Docker image is a way to ensure the best version of Python is always used. + +**Important:** This release requires a new IAM permission, ``servicequotas:ListServiceQuotas``. + +* `Issue #400 `__ / `PR #434 `__ - Support GovCloud region and alternate partitions in STS assumed roles and Trusted Advisor. Thanks to `@djkiourtsis `__. +* `Issue #432 `__ - Update EC2 limit handling for new vCPU-based limits in regions other than ``cn-*`` and ``us-gov-*`` (which still use old per-instance-type limits). See :ref:`section below ` for further information. For regions other than ``cn-*`` and ``us-gov-*``, **this will remove** all 175 ``Running On-Demand instances`` and the ``Running On-Demand EC2 instances`` limit, and replace them with: + + * ``Running On-Demand All F instances`` + * ``Running On-Demand All G instances`` + * ``Running On-Demand All P instances`` + * ``Running On-Demand All X instances`` + * ``Running On-Demand All Standard (A, C, D, H, I, M, R, T, Z) instances`` + +* `Issue #429 `_ - add 87 missing EC2 instance types. This will now only impact ``cn-*`` and ``us-gov-*`` regions. +* `Issue #433 `_ - Fix broken links in the docs; waffle.io and landscape.io are both gone, sadly. +* `Issue #441 `_ - Fix critical bug where awslimitchecker would die with an unhandled ``botocore.exceptions.ParamValidationError`` exception in accounts that have Trusted Advisor but do not have a "Service Limits" check in the "performance" category. +* `Issue #439 `_ - Fix unhandled exception in CloudTrail service when attempting to call ``GetEventSelectors`` on an Organization trail. When calling ``DescribeTrails``, we will now pass ``includeShadowTrails`` as False, to not include replications of trails in different regions or organization trails in member accounts (relevant `API documentation `_). +* `Issue #438 `_ - Per `PEP 373 `__, Python 2.7 will officially end support on January 1, 2020. As such, and in keeping with reasoning explained at `python3statement.org `__, awslimitchecker will **stop supporting and testing against Python 2.7** on January 1, 2020. At that point, all new versions will be free to use Python features introduced in 3.5. As of this version, a `PendingDeprecationWarning `__ will be emitted when running awslimitchecker under Python 2.7. +* `Issue #437 `_ - Per `PEP 429 `_, Python 3.4 reached end-of-life on March 18, 2019 and is now officially retired. Add a ``PendingDeprecationWarning`` for users running under this version, announcing that support for Python 3.4 will be removed on January 1, 2020. +* In following with the above two issues, raise a ``DeprecationWarning`` when running on any Python2 version prior to 2.7 or any Python3 version prior to 3.4, in accorance with the `published end-of-life dates of those versions `_. +* `Issue #436 `_ - Begin testing under Python 3.8 and base our Docker image on ``python:3.8-alpine``. +* `Issue #435 `_ - Allow configuring the botocore maximum retries for Throttling / RateExceeded errors on a per-AWS-API basis via environment variables. See the relevant sections of the :ref:`CLI Usage ` or :ref:`Python Usage ` documentation for further details. +* `Issue #431 `_ - Fix a **major under-calculation** of usage for the EC2 ``Rules per VPC security group`` limit. We were previously calculating the number of "Rules" (from port / to port / protocol combinations) in a Security Group, but the limit is actually based on the number of permissions granted. See `this comment `_ on the issue for further details. +* `Issue #413 `_ - Add support for retrieving limits from the new `Service Quotas service `__ where available. See the :ref:`changelog.8_0_0_service_quotas` section below for more information. +* Bump boto3 minimum version requirement from 1.4.6 to 1.9.175 and botocore minimum version requirement from 1.6.0 to 1.12.175, in order to support Service Quotas. + +.. _changelog.8_0_0_vcpu_limits: + +New EC2 vCPU Limits ++++++++++++++++++++ + +AWS has `announced `__ new, completely different handling of EC2 On-Demand Instances service limits. Instead of having a limit per instance type (currently 261 limits), there will now be only *five* limits, based on the number of vCPUs for instance families: one each for "F", "G", "P", and "X" family instances (defaulting to a total of 128 vCPUs each) and one limit for all other "Standard" instance families (currently A, C, D, H, I, M, R, T, and Z) defaulting to a combined total of 1152 vCPUs. Please see the link, and the `EC2 On-Demand Instance Limits section of the AWS FAQ `__ for further information. + +This greatly simplifies handling of the EC2 On-Demand limits, but does mean that any existing code that references EC2 Running On-Demand limit names, including any limit and/or threshold overrides, will need to be updated for this change. + +This change is only going into effect in the "standard" AWS regions/partitions, i.e. not in the China partition (``cn-`` regions) or GovCloud (``us-gov-`` regions). It is a phased rollout from October 24 to November 7, 2019 based on the first character of your account ID (see the "How will the transition to vCPU limits happen?" entry in the FAQ linked above for exact dates). **Unfortunately, there is no clear way to determine via API if a given account is using the new vCPU limits or the old per-instance-type limits.** As a result, and given that this release is being made already part-way through the rollout window, the current behavior of awslimitchecker is as follows: + +* When running against region names beginning with ``cn-`` or ``us-gov-``, use the old per-instance-type limits, unless the ``USE_VCPU_LIMITS`` environment variable is set to ``true``. +* Otherwise use the new vCPU-based limits, unless the ``USE_VCPU_LIMITS`` environment variable is set to something other than ``true``. + +As such, if you install this release before November 7, 2019 and need to force your non-China, non-GovCloud accout to use the older per-instance-type limits, setting the ``USE_VCPU_LIMITS`` environment variable to ``false`` will accomplish this until your account switches over to the new vCPU limits. **Alternatively, you can leave awslimitchecker as-is and accept possibly-slightly-inaccurate limit calculations for a few days.** + +Please also note that with the change to vCPU limits, there is no longer an overall ``Running On-Demand EC2 instances`` limit for accounts that use the new vCPU limits. + +I have **not** yet implemented Trusted Advisor (TA) support for these new limits, as they're presented in a different category of Trusted Advisor checks from the previous EC2 limits. I'm not going to be implementing TA for these limits, in favor of spending the time instead on implementing Service Quotas support via `Issue #413 `__. + +Calculation of current usage for the vCPU limits is based on the `EC2 Optimizing CPU Options documentation `__ which specifies, "The number of vCPUs for the instance is the number of CPU cores multiplied by the threads per core." The ``CpuOptions`` field of the EC2 ``DescribeInstances`` API specifies the core and thread count for each running instance. + +.. _changelog.8_0_0_service_quotas: + +Service Quotas +++++++++++++++ + +AWS' new `Service Quotas service `__ provides a unified interface to retrieve current limits from many AWS services. These limit values are second only to the services' own APIs (for the services that provide limit information via API), and are much more current and complete than the information provided by Trusted Advisor. The introduction of Service Quotas should greatly reduce the number of limits that need to be retrieved from Trusted Advisor or specified manually. + +If you currently have any Limit Overrides set (via either the :ref:`CLI ` or :ref:`Python API `), please verify on the :ref:`limits` page whether Service Quotas data is now available for those limits. You should be able to remove manual overrides for the limits that now retrieve data from Service Quotas. + +.. _changelog.7_1_0: + 7.1.0 (2019-09-10) ------------------ @@ -14,6 +80,8 @@ Changelog * `Issue #418 `__ - Add support for sending runtime, limits, and usage to :ref:`` such as Datadog. * `Issue #419 `__ - Add support for alerts/notifications of thresholds crossed or failed runs (exceptions) via :ref:`` such as PagerDuty. +.. _changelog.7_0_0: + 7.0.0 (2019-08-13) ------------------ @@ -23,51 +91,71 @@ This release **removes one limit and adds two new limits**! * `Issue #410 `__ - Documentation fix for missing Trusted Advisor information on Limits page. * Fix some test failures related to exception objects in pytest 5.0.0. +.. _changelog.6_1_7: + 6.1.7 (2019-05-17) ------------------ * `Issue #406 `__ - Fix for unhandled exception when a Trusted Advisor check has a ``null`` timestamp. +.. _changelog.6_1_6: + 6.1.6 (2019-04-19) ------------------ * `PR #402 `__ - Add ``--skip-check`` command line option for ignoring specific checks based on service and check name. Thanks to `@ddelnano `__. +.. _changelog.6_1_5: + 6.1.5 (2019-03-06) ------------------ * `Issue #397 `__ - Fix unhandled exception checking SES in some regions. `Issue #375 `__ in 6.0.1 handled an uncaught ``ClientError`` when checking SES in some regions, but some regions such as ap-southeast-2 are now returning a 503 Service Unavailable for SES instead. Handle this case as well. Thanks to `@TimGebert `__ for reporting the issue and `bergkampsliew `__ for verifying. +.. _changelog.6_1_4: + 6.1.4 (2019-03-01) ------------------ * `PR #394 `_ - Fix bug in calculation of VPC "Network interfaces per Region" limit, added in 6.1.0 (`PR #379 `__), that resulted in reporting the limit 5x lower than it actually is in some cases. Thanks to `@TimGebert `__. +.. _changelog.6_1_3: + 6.1.3 (2019-02-26) ------------------ * `PR #391 `_ / `Issue #390 `_ - Update for some recently-increased DynamoDB and EFS default limits. Thanks to `bergkampsliew `__. +.. _changelog.6_1_2: + 6.1.2 (2019-02-19) ------------------ * `PR #387 `_ - Fix bug in calculation of VPC "Network interfaces per Region" limit, added in 6.1.0 (`PR #379 `__). Thanks to `@nadlerjessie `__. +.. _changelog.6_1_1: + 6.1.1 (2019-02-15) ------------------ * `PR #381 `_ / `Issue #382 `_ - Revised fix for `Issue #375 `__, uncaught ``ClientError`` exception when checking SES Send Quota in certain regions. Thanks to `bergkampsliew `__. +.. _changelog.6_1_0: + 6.1.0 (2019-01-30) ------------------ * `PR #379 `__ - Add support for EC2/VPC ``Network interfaces per Region`` limit. Thanks to `@nadlerjessie `__. +.. _changelog.6_0_1: + 6.0.1 (2019-01-27) ------------------ * `Issue #375 `__ - Fix uncaught ``ClientError`` exception when checking SES Send Quota in certain regions. Thanks to `bergkampsliew `__ for `PR #376 `_. +.. _changelog.6_0_0: + 6.0.0 (2019-01-01) ------------------ @@ -97,6 +185,8 @@ awslimitchecker has had documented support for Limits that are unlimited/"infini If you are relying on the output format of the command line ``awslimitchecker`` script, please use the Python API instead. +.. _changelog.5_1_0: + 5.1.0 (2018-09-23) ------------------ @@ -104,6 +194,8 @@ If you are relying on the output format of the command line ``awslimitchecker`` * `PR #359 `_ - Add support for ``t3`` EC2 instance types (thanks to `chafouin `_). * Switch ``py37`` TravisCI tests from py37-dev to py37 (release). +.. _changelog.5_0_0: + 5.0.0 (2018-07-30) ------------------ @@ -150,6 +242,8 @@ If you are relying on the output format of the command line ``awslimitchecker`` For users of the Python API, please take note of the new :py:meth:`.AwsLimit.has_resource_limits` and :py:meth:`~.AwsLimitUsage.get_maximum` methods which assist in how to identify limits that have per-resource maxima. Existing code that only surfaces awslimitchecker's warnings/criticals (the result of :py:meth:`~.AwsLimitChecker.check_thresholds`) will work without modification, but any code that displays or uses the current limit values themselves may need to be updated. +.. _changelog.4_0_2: + 4.0.2 (2018-03-22) ------------------ @@ -157,6 +251,8 @@ This is a minor bugfix release for one issue: * `Issue #341 `_ - The Trusted Advisor EBS checks for ``General Purpose (SSD) volume storage (GiB)`` and ``Magnetic volume storage (GiB)`` have been renamed to to ``General Purpose SSD (gp2) volume storage (GiB)`` and ``Magnetic (standard) volume storage (GiB)``, respectively, to provide more unified naming. This change was made on March 19th or 20th without any public announcement, and resulted in awslimitchecker being unable to determine the current values for these limits from Trusted Advisor. Users relying on Trusted Advisor for these values saw the limit values incorrectly revert to the global default. This is an internal-only change to map the new Trusted Advisor check names to the awslimitchecker limit names. +.. _changelog.4_0_1: + 4.0.1 (2018-03-09) ------------------ @@ -165,6 +261,8 @@ This is a minor bugfix release for a few issues that users have reported recentl * Fix `Issue #337 `_ where sometimes an account even with Business-level support will not have a Trusted Advisor result for the Service Limits check, and will return a result with ``status: not_available`` or a missing ``flaggedResources`` key. * Fix `Issue #335 `_ where runs against the EFS service in certain unsupported regions result in either a connection timeout or an AccessDeniedException. +.. _changelog.4_0_0: + 4.0.0 (2018-02-17) ------------------ @@ -212,6 +310,8 @@ This release **requires new IAM permissions**: * Fix date and incorrect project name in some file/copyright headers. * `Issue #331 `_ - Change layout of the generated `Supported Limits `_ documentation page to be more clear about which limits are supported, and include API and Trusted Advisor data in the same table as the limits and their defaults. +.. _changelog.3_0_0: + 3.0.0 (2017-12-02) ------------------ @@ -228,6 +328,8 @@ after development was ceased. The test framework used by awslimitchecker, pytest * `Issue #315 `_ - Add new instance types: 'c5.18xlarge', 'c5.2xlarge', 'c5.4xlarge', 'c5.9xlarge', 'c5.large', 'c5.xlarge', 'g3.16xlarge', 'g3.4xlarge', 'g3.8xlarge', 'h1.16xlarge', 'h1.2xlarge', 'h1.4xlarge', 'h1.8xlarge', 'm5.12xlarge', 'm5.24xlarge', 'm5.2xlarge', 'm5.4xlarge', 'm5.large', 'm5.xlarge', 'p3.16xlarge', 'p3.2xlarge', 'p3.8xlarge', 'x1e.32xlarge', 'x1e.xlarge' * `Issue #316 `_ - Automate release process. +.. _changelog.2_0_0: + 2.0.0 (2017-10-12) ------------------ @@ -235,6 +337,8 @@ after development was ceased. The test framework used by awslimitchecker, pytest * Update minimum ``boto3`` version requirement from 1.2.3 to 1.4.4; the code for `Issue #268 `_ released in 0.11.0 requires boto3 >= 1.4.4 to make the ElasticLoadBalancing ``DescribeAccountLimits`` call. * **Bug fix for "Running On-Demand EC2 instances" limit** - `Issue #308 `_ - The fix for `Issue #215 `_ / `PR #223 `_, released in 0.6.0 on November 11, 2016 was based on `incorrect information `_ about how Regional Benefit Reserved Instances (RIs) impact the service limit. The code implemented at that time subtracted Regional Benefit RIs from the count of running instances that we use to establish usage. Upon further review, as well as confirmation from AWS Support, some AWS TAMs, and the `relevant AWS documentation `_, only Zonal RIs (AZ-specific) are exempt from the Running On-Demand Instances limit. Regional Benefit RIs are counted the same as any other On-Demand Instances, as they don't have reserved capacity. This release stops subtracting Regional Benefit RIs from the count of Running Instances, which was causing awslimitchecker to report inaccurately low Running Instances usage. +.. _changelog.1_0_0: + 1.0.0 (2017-09-21) ------------------ @@ -258,6 +362,8 @@ Changes in this release: * `PR #302 `_ - Add support for VPC VPN Gateways limit. (Thanks to `andrewmichael `_ for the contribution.) * `Issue #280 `_ / `PR #297 `_ - Add support for DynamoDB limits. (Thanks to `saratlingamarla `_ for the contribution.) +.. _changelog.0_11_0: + 0.11.0 (2017-08-06) ------------------- @@ -274,6 +380,8 @@ Changes in this release: * `Issue #287 `_ / `PR #288 `_ - Add support for Elastic Filesystem number of filesystems limit. (Thanks to `nicksantamaria `_ for the contribution.) * `Issue #268 `_ - Add support for ELBv2 (Application Load Balancer) limits; get ELBv1 (Classic) and ELBv2 (Application) limits from the DescribeAccountLimits API calls. +.. _changelog.0_10_0: + 0.10.0 (2017-06-25) ------------------- @@ -289,6 +397,8 @@ This release **removes the ElastiCache Clusters limit**, which no longer exists. * `Issue #279 `_ - Add Github release to release process. +.. _changelog.0_9_0: + 0.9.0 (2017-06-11) ------------------ @@ -313,6 +423,8 @@ This release **removes the ElastiCache Clusters limit**, which no longer exists. available in ``us-east-1``, ``us-west-2`` and ``eu-west-1``. Omit the traceback from the log message for Firehose ``EndpointConnectionError`` and log at warning instead of error. +.. _changelog.0_8_0: + 0.8.0 (2017-03-11) ------------------ @@ -351,6 +463,8 @@ or bug reports specific to 3.2 will be closed. * `Issue #200 `_ - Remove EC2 Spot Instances/Fleets limits from experimental status. * `Issue #123 `_ - Update documentation on using session tokens (Session or Federation temporary creds). +.. _changelog.0_7_0: + 0.7.0 (2017-01-15) ------------------ @@ -406,6 +520,8 @@ See `Getting Started - Trusted Advisor `_ - fix KeyError when ``timestamp`` key is missing from TrustedAdvisor check result dict +.. _changelog.0_5_0: + 0.5.0 (2016-07-06) ------------------ @@ -485,16 +605,22 @@ This release includes a change to ``awslimitchecker``'s Python API. `awslimitche * `#195 `_ - Handle TrustedAdvisor explicitly reporting some limits as "unlimited". This introduces the concept of unlimited limits, where the effective limit is ``None``. +.. _changelog.0_4_4: + 0.4.4 (2016-06-27) ------------------ * `PR #190 `_ / `#189 `_ - Add support for EBS st1 and sc1 volume types (adds "EBS/Throughput Optimized (HDD) volume storage (GiB)" and "EBS/Cold (HDD) volume storage (GiB)" limits). +.. _changelog.0_4_3: + 0.4.3 (2016-05-08) ------------------ * `PR #184 `_ Fix default VPC/Security groups per VPC limit from 100 to 500, per `VPC limits documentation `_ (this limit was increased at some point recently). Thanks to `Travis Thieman `_ for this contribution. +.. _changelog.0_4_2: + 0.4.2 (2016-04-27) ------------------ @@ -509,11 +635,15 @@ This release requires the following new IAM permissions to function: * `#175 `_ the simplest and most clear contributor license agreement I could come up with. * `#172 `_ add an integration test running against sa-east-1, which has fewer services than the popular US regions. +.. _changelog.0_4_1: + 0.4.1 (2016-03-15) ------------------ * `#170 `_ Critical bug fix in implementation of `#71 `_ - SES only supports three regions (us-east-1, us-west-2, eu-west-1) and causes an unhandled connection error if used in another region. +.. _changelog.0_4_0: + 0.4.0 (2016-03-14) ------------------ @@ -542,11 +672,15 @@ Issues addressed: * `#69 `_ Add support for CloudFormation service Stacks limit. This **requires additional IAM permissions**, ``cloudformation:DescribeAccountLimits`` and ``cloudformation:DescribeStacks``. * `#166 `_ Speed up TravisCI tests by dropping testing for PyPy and PyPy3, and only running the -versioncheck tests for two python interpreters instead of 8. +.. _changelog.0_3_2: + 0.3.2 (2016-03-11) ------------------ * `#155 `_ Bug fix for uncaught KeyError on accounts with Trusted Advisor (business-level support and above). This was caused by an undocumented change released by AWS between Thu, 10 Mar 2016 07:00:00 GMT and Fri, 11 Mar 2016 07:00:00 GMT, where five new IAM-related checks were introduced that lack the ``region`` data field (which the `TrustedAdvisorResourceDetail API docs `_ still list as a required field). +.. _changelog.0_3_1: + 0.3.1 (2016-03-04) ------------------ @@ -556,6 +690,8 @@ Issues addressed: * `#128 `_ Update Development and Getting Help documentation; add GitHub CONTRIBUTING.md file with link back to docs, as well as Issue and PR templates. * `#131 `_ Refactor TrustedAdvisor interaction with limits for special naming cases (limits where the TrustedAdvisor service or limit name doesn't match that of the awslimitchecker limit); enable newly-available TrustedAdvisor data for some EC2 on-demand instance usage. +.. _changelog.0_3_0: + 0.3.0 (2016-02-18) ------------------ @@ -568,24 +704,32 @@ Issues addressed: * `#114 `_ expanded automatic integration tests * **Please note** that version 0.3.0 of awslimitchecker moved from using ``boto`` as its AWS API client to using ``boto3``. This change is mostly transparent, but there is a minor change in how AWS credentials are handled. In ``boto``, if the ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY`` environment variables were set, and the region was not set explicitly via awslimitchecker, the AWS region would either be taken from the ``AWS_DEFAULT_REGION`` environment variable or would default to us-east-1, regardless of whether a configuration file (``~/.aws/credentials`` or ``~/.aws/config``) was present. With boto3, it appears that the default region from the configuration file will be used if present, regardless of whether the credentials come from that file or from environment variables. +.. _changelog.0_2_3: + 0.2.3 (2015-12-16) ------------------ * `PR #100 `_ support MFA tokens when using STS assume role * `#107 `_ add support to explicitly disable pagination, and use for TrustedAdvisor to prevent pagination warnings +.. _changelog.0_2_2: + 0.2.2 (2015-12-02) ------------------ * `#83 `_ remove the "v" prefix from version tags so ReadTheDocs will build them automatically. * `#21 `_ run simple integration tests of ``-l`` and ``-u`` for commits to main repo branches. +.. _changelog.0_2_1: + 0.2.1 (2015-12-01) ------------------ * `#101 `_ Ignore stopped and terminated instances from EC2 Running On-Demand Instances usage count. * `#47 `_ In VersionCheck git -e tests, explicitly fetch git tags at beginning of test. +.. _changelog.0_2_0: + 0.2.0 (2015-11-29) ------------------ @@ -606,6 +750,8 @@ Issues addressed: * Add ``autoscaling:DescribeAccountLimits`` and ``ec2:DescribeAccountAttributes`` to required IAM permissions. * Ignore ``AccountLimits`` objects from result pagination +.. _changelog.0_1_3: + 0.1.3 (2015-10-04) ------------------ @@ -625,11 +771,15 @@ Issues addressed: * `#76 `_ default limits for EBS volume usage were in TiB not GiB, causing invalid default limits on accounts without Trusted Advisor * Changes to some tests in ``test_versioncheck.py`` to aid in debugging `#47 `_ where Travis tests fail on master because of git tag from release (if re-run after release) +.. _changelog.0_1_2: + 0.1.2 (2015-08-13) ------------------ * `#62 `_ - For 'RDS/DB snapshots per user' limit, only count manual snapshots. (fix bug in fix for `#54 `_) +.. _changelog.0_1_1: + 0.1.1 (2015-08-13) ------------------ @@ -645,6 +795,8 @@ Issues addressed: * move boto log suppression from checker to runner * Add contributing docs +.. _changelog.0_1_0: + 0.1.0 (2015-07-25) ------------------ diff --git a/Dockerfile b/Dockerfile index 370e977f..fcd96da4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-alpine +FROM python:3.8-alpine ARG git_version diff --git a/README.rst b/README.rst index ce8b6325..4cd7bb57 100644 --- a/README.rst +++ b/README.rst @@ -17,10 +17,6 @@ awslimitchecker :alt: GitHub Open Issues :target: https://github.com/jantman/awslimitchecker/issues -.. image:: https://badge.waffle.io/jantman/awslimitchecker.png?label=ready&title=Ready - :target: https://waffle.io/jantman/awslimitchecker - :alt: 'Stories in Ready - waffle.io' - .. image:: http://www.repostatus.org/badges/1.1.0/active.svg :alt: Project Status: Active - The project has reached a stable, usable state and is being actively developed. :target: http://www.repostatus.org/#active @@ -35,10 +31,6 @@ Master: :target: http://travis-ci.org/jantman/awslimitchecker :alt: travis-ci for master branch -.. image:: https://landscape.io/github/jantman/awslimitchecker/master/landscape.svg?style=flat - :target: https://landscape.io/github/jantman/awslimitchecker/master - :alt: Code Health - .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=master :target: https://codecov.io/github/jantman/awslimitchecker?branch=master :alt: coverage report for master branch @@ -53,10 +45,6 @@ Develop: :target: http://travis-ci.org/jantman/awslimitchecker :alt: travis-ci for develop branch -.. image:: https://landscape.io/github/jantman/awslimitchecker/develop/landscape.svg?style=flat - :target: https://landscape.io/github/jantman/awslimitchecker/develop - :alt: Code Health - .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=develop :target: https://codecov.io/github/jantman/awslimitchecker?branch=develop :alt: coverage report for develop branch @@ -70,9 +58,11 @@ A script and python module to check your AWS service limits and usage, and warn Users building out scalable services in Amazon AWS often run into AWS' `service limits `_ - often at the least convenient time (i.e. mid-deploy or when autoscaling fails). Amazon's `Trusted Advisor `_ can help this, but even the version that comes with Business and Enterprise support only monitors a small subset of AWS limits -and only alerts *weekly*. awslimitchecker provides a command line script and reusable package that queries your current -usage of AWS resources and compares it to limits (hard-coded AWS defaults that you can override, API-based limits where available, or data from Trusted -Advisor where available), notifying you when you are approaching or at your limits. +and only alerts *weekly*. The new Service Quotas service can also help with this, but relies on CloudWatch alarms per-limit to notify +you when you approach your limits; this cannot easily scale to the hundreds of current service limits. awslimitchecker provides a command line +script and reusable Python package that queries your current usage of AWS resources and compares it to limits (hard-coded AWS defaults that you +can override, API-based limits where available, Service Quotas data where available, or data from Trusted Advisor where available), notifying +you when you are approaching or at your limits. Full project documentation for the latest release is available at `http://awslimitchecker.readthedocs.io/en/latest/ `_. @@ -94,6 +84,7 @@ What It Does - Define custom thresholds per-limit - where possible, pull current limits from Trusted Advisor API - where possible, pull current limits from each service's API (for services that provide this information) +- where possible, pull current limits from the Service Quotas service - Supports explicitly setting the AWS region - Supports using `STS `_ to assume roles in other accounts, including using ``external_id``. - Optionally refresh Trusted Advisor "Service Limits" check before polling @@ -109,7 +100,7 @@ Requirements **Either Docker in order to run via the** `docker image `__, **or:** -* Python 2.7 or 3.4+. Python 2.6 and 3.3 are no longer supported. +* Python 3.5 or newer. Python 2.7 will not be supported as of January 1, 2010. * Python `VirtualEnv `_ and ``pip`` (recommended installation method; your OS/distribution should have packages for these) * `boto3 `_ >= 1.4.6 and its dependency `botocore `_ >= 1.6.0. diff --git a/awslimitchecker/checker.py b/awslimitchecker/checker.py index 057acb16..395115d9 100644 --- a/awslimitchecker/checker.py +++ b/awslimitchecker/checker.py @@ -42,20 +42,30 @@ from .trustedadvisor import TrustedAdvisor from .version import _get_version_info from .utils import _get_latest_version +from .quotas import ServiceQuotasClient import boto3 import sys import logging +import warnings logger = logging.getLogger(__name__) +warnings.filterwarnings( + action="always", category=DeprecationWarning, module=__name__ +) +warnings.filterwarnings( + action="always", category=PendingDeprecationWarning, module=__name__ +) + class AwsLimitChecker(object): def __init__(self, warning_threshold=80, critical_threshold=99, profile_name=None, account_id=None, account_role=None, - region=None, external_id=None, mfa_serial_number=None, - mfa_token=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True): + role_partition='aws', region=None, external_id=None, + mfa_serial_number=None, mfa_token=None, ta_refresh_mode=None, + ta_refresh_timeout=None, ta_api_region='us-east-1', + check_version=True, skip_quotas=False): """ Main AwsLimitChecker class - this should be the only externally-used portion of awslimitchecker. @@ -89,6 +99,10 @@ def __init__(self, warning_threshold=80, critical_threshold=99, :param region: AWS region name to connect to :type region: str :type account_role: str + :param role_partition: `AWS role partition `_ + for the account_role to connect via STS + :type role_partition: str :param external_id: (optional) the `External ID `_ string to use when assuming a role via STS. @@ -116,9 +130,16 @@ def __init__(self, warning_threshold=80, critical_threshold=99, parameter is not None, only wait up to this number of seconds for the refresh to finish before continuing on anyway. :type ta_refresh_timeout: :py:class:`int` or :py:data:`None` + :param ta_api_region: The AWS region used for calls to the + TrustedAdvisor API. This is always us-east-1 for + non GovCloud accounts. + :type ta_api_region: str :param check_version: Whether or not to check for latest version of awslimitchecker on PyPI during instantiation. :type check_version: bool + :param skip_quotas: If set to True, do not connect to Service Quotas + service or use it to obtain current limits. + :type skip_quotas: bool """ # ###### IMPORTANT license notice ########## # Pursuant to Sections 5(b) and 13 of the GNU Affero General Public @@ -151,11 +172,13 @@ def __init__(self, warning_threshold=80, critical_threshold=99, ' is %s; please consider upgrading.', self.vinfo.release, latest_ver ) + self._check_python_version() self.warning_threshold = warning_threshold self.critical_threshold = critical_threshold self.profile_name = profile_name self.account_id = account_id self.account_role = account_role + self.role_partition = role_partition self.external_id = external_id self.mfa_serial_number = mfa_serial_number self.mfa_token = mfa_token @@ -164,15 +187,64 @@ def __init__(self, warning_threshold=80, critical_threshold=99, self.services = {} boto_conn_kwargs = self._boto_conn_kwargs + self._quotas_client = None + if not skip_quotas: + self._quotas_client = ServiceQuotasClient(boto_conn_kwargs) for sname, cls in _services.items(): self.services[sname] = cls(warning_threshold, critical_threshold, - boto_conn_kwargs) + boto_conn_kwargs, + self._quotas_client) self.ta = TrustedAdvisor(self.services, boto_conn_kwargs, ta_refresh_mode=ta_refresh_mode, - ta_refresh_timeout=ta_refresh_timeout) + ta_refresh_timeout=ta_refresh_timeout, + ta_api_region=ta_api_region) + + def _check_python_version(self): + """ + Check that we are running under a supported Python version, and emit a + warning otherwise. + """ + if sys.version_info[:2] == (2, 7): # nocoverage + warnings.warn( + 'awslimitchecker has detected that it is running under Python ' + '2.7. This will no longer be supported as of January 1, 2020. ' + 'Please update to a newer Python version (>= 3.5) or switch ' + 'to running via the official Docker image. For further ' + 'information, please see the awslimitchecker 8.0.0 changelog ' + 'at ', + PendingDeprecationWarning + ) + elif sys.version_info[:2] == (3, 4): # nocoverage + warnings.warn( + 'awslimitchecker has detected that it is running under Python ' + '3.4. This will no longer be supported as of January 1, 2020. ' + 'Please update to a newer Python version (>= 3.5) or switch ' + 'to running via the official Docker image. For further ' + 'information, please see the awslimitchecker 8.0.0 changelog ' + 'at ', + PendingDeprecationWarning + ) + elif ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 + ): # nocoverage + warnings.warn( + 'awslimitchecker has detected that it is running under Python ' + '%d.%d. This version has reached end-of-life and is no longer ' + 'supported by awslimitchecker, and may not function correctly. ' + 'Please update to a newer Python version (>= 3.5) or switch ' + 'to running via the official Docker image. For further ' + 'information, please see the awslimitchecker 8.0.0 changelog ' + 'at ' + '' % (sys.version_info[0], sys.version_info[1]), + DeprecationWarning + ) @property def _boto_conn_kwargs(self): @@ -277,6 +349,7 @@ def get_limits(self, service=None, use_ta=True): for sname, cls in to_get.items(): if hasattr(cls, '_update_limits_from_api'): cls._update_limits_from_api() + cls._update_service_quotas() res[sname] = cls.get_limits() return res @@ -306,7 +379,11 @@ def _get_sts_token(self): """ logger.debug("Connecting to STS in region %s", self.region) sts = boto3.client('sts', region_name=self.region) - arn = "arn:aws:iam::%s:role/%s" % (self.account_id, self.account_role) + arn = "arn:%s:iam::%s:role/%s" % ( + self.role_partition, + self.account_id, + self.account_role + ) logger.debug("STS assume role for %s", arn) assume_kwargs = { 'RoleArn': arn, @@ -351,6 +428,7 @@ def find_usage(self, service=None, use_ta=True): for cls in to_get.values(): if hasattr(cls, '_update_limits_from_api'): cls._update_limits_from_api() + cls._update_service_quotas() logger.debug("Finding usage for service: %s", cls.service_name) cls.find_usage() @@ -551,6 +629,7 @@ def check_thresholds(self, service=None, use_ta=True): for sname, cls in to_get.items(): if hasattr(cls, '_update_limits_from_api'): cls._update_limits_from_api() + cls._update_service_quotas() tmp = cls.check_thresholds() if len(tmp) > 0: res[sname] = tmp @@ -569,6 +648,7 @@ def get_required_iam_policy(self): :rtype: dict """ required_actions = [ + 'servicequotas:ListServiceQuotas', 'support:*', 'trustedadvisor:Describe*', 'trustedadvisor:RefreshCheck' diff --git a/awslimitchecker/connectable.py b/awslimitchecker/connectable.py index b14aae9f..55cf7592 100644 --- a/awslimitchecker/connectable.py +++ b/awslimitchecker/connectable.py @@ -37,8 +37,10 @@ ################################################################################ """ +import os import logging import boto3 +from botocore.config import Config logger = logging.getLogger(__name__) @@ -66,12 +68,38 @@ def __init__(self, creds_dict): class Connectable(object): - """ Mix-in helper class for connecting to AWS APIs. Centralizes logic of connecting via regions and/or STS. """ + @property + def _max_retries_config(self): + """ + If a ``BOTO_MAX_RETRIES_`` environment variable is set, + return a new ``botocore.config.Config`` instance using that number + as the retries max_attempts value. + + :rtype: ``botocore.config.Config`` or None + """ + key = 'BOTO_MAX_RETRIES_%s' % self.api_name + if key not in os.environ: + return None + try: + max_retries = int(os.environ[key]) + except Exception: + logger.error( + 'ERROR: Found "%s" environment variable, but unable to ' + 'parse value "%s" to an integer.', key, os.environ[key] + ) + return None + logger.debug( + 'Setting explicit botocore retry config with max_attempts=%d ' + 'for "%s" API based on %s environment variable.', + max_retries, self.api_name, key + ) + return Config(retries={'max_attempts': max_retries}) + def connect(self): """ Connect to an AWS API via boto3 low-level client and set ``self.conn`` @@ -84,7 +112,9 @@ def connect(self): """ if self.conn is not None: return - kwargs = self._boto3_connection_kwargs + kwargs = dict(self._boto3_connection_kwargs) + if self._max_retries_config is not None: + kwargs['config'] = self._max_retries_config self.conn = boto3.client(self.api_name, **kwargs) logger.info("Connected to %s in region %s", self.api_name, self.conn._client_config.region_name) @@ -102,7 +132,9 @@ def connect_resource(self): """ if self.resource_conn is not None: return - kwargs = self._boto3_connection_kwargs + kwargs = dict(self._boto3_connection_kwargs) + if self._max_retries_config is not None: + kwargs['config'] = self._max_retries_config self.resource_conn = boto3.resource(self.api_name, **kwargs) logger.info("Connected to %s (resource) in region %s", self.api_name, self.resource_conn.meta.client._client_config.region_name) diff --git a/awslimitchecker/limit.py b/awslimitchecker/limit.py index 0161762b..29d4682e 100644 --- a/awslimitchecker/limit.py +++ b/awslimitchecker/limit.py @@ -49,13 +49,18 @@ #: indicates a limit value that came from the service's API SOURCE_API = 3 +#: indicates a limit value that came from the Service Quotas service +SOURCE_QUOTAS = 4 + class AwsLimit(object): def __init__(self, name, service, default_limit, def_warning_threshold, def_critical_threshold, limit_type=None, limit_subtype=None, - ta_service_name=None, ta_limit_name=None): + ta_service_name=None, ta_limit_name=None, + quotas_service_code=None, quotas_name=None, + quotas_unit='None', quotas_unit_converter=None): """ Describes one specific AWS service limit, as well as its current utilization, default limit, thresholds, and any @@ -88,6 +93,28 @@ def __init__(self, name, service, default_limit, :param ta_limit_name: The limit name returned by Trusted Advisor for this limit, if different from ``name``. :type ta_limit_name: str + :param quotas_service_code: the Service Quotas service code to + retrieve this limit from, if different from the + :py:attr:`~._AwsService.quotas_service_code` attribute of + ``service``. + :type quotas_service_code: str or None + :param quotas_name: the Service Quotas quota name to use for this + limit, if different from the limit ``name``. + :type quotas_name: str or None + :param quotas_unit: the Service Quotas quota unit that we need our + limit value to be, for quotas that use units. This must be one of + the units supported by :py:class:`~.ServiceQuotasClient`. It defaults + to the string "None", which (for some strange reason) is what's + returned by the Service Quotas API. + :type quotas_unit: str + :param quotas_unit_converter: A callable to be passed to + :py:meth:`~.ServiceQuotasClient.get_quota_value` for unit conversion. + Must take three positional arguments: the Service Quotas value for + this quota (float), the quota ``Unit`` str, and the return value of + :py:meth:`~.quotas_unit`. This callable is responsible for converting + the quota value from the quota Unit to this class's expected unit. + If they cannot be converted, it should log an error and return None. + :type quotas_unit_converter: ``callable`` :raises: ValueError """ if def_warning_threshold >= def_critical_threshold: @@ -114,6 +141,11 @@ def __init__(self, name, service, default_limit, self._criticals = [] self._ta_service_name = ta_service_name self._ta_limit_name = ta_limit_name + self._quotas_service_code = quotas_service_code + self._quotas_name = quotas_name + self._quotas_unit = quotas_unit + self.quotas_limit = None + self.quotas_unit_converter = quotas_unit_converter def set_limit_override(self, limit_value, override_ta=True): """ @@ -162,20 +194,34 @@ def _set_api_limit(self, limit_value): """ self.api_limit = limit_value + def _set_quotas_limit(self, limit_value): + """ + Set the value for the limit as reported by the Service Quotas service. + + This method should only be called from the Service class. + + :param limit_value: the Service Quotas limit value + :type limit_value: float + """ + self.quotas_limit = limit_value + def get_limit_source(self): """ - Return :py:const:`~awslimitchecker.limit.SOURCE_DEFAULT` if + Return :py:data:`~awslimitchecker.limit.SOURCE_DEFAULT` if :py:meth:`~.get_limit` returns the default limit, - :py:const:`~awslimitchecker.limit.SOURCE_OVERRIDE` if it returns a + :py:data:`~awslimitchecker.limit.SOURCE_OVERRIDE` if it returns a manually-overridden limit, - :py:const:`~awslimitchecker.limit.SOURCE_TA` if it returns a limit from - Trusted Advisor, or :py:const:`~awslimitchecker.limit.SOURCE_API` - if it returns a limit retrieved from the service's API. - - :returns: one of :py:const:`~awslimitchecker.limit.SOURCE_DEFAULT`, - :py:const:`~awslimitchecker.limit.SOURCE_OVERRIDE`, or - :py:const:`~awslimitchecker.limit.SOURCE_TA`, or - :py:const:`~awslimitchecker.limit.SOURCE_API` + :py:data:`~awslimitchecker.limit.SOURCE_TA` if it returns a limit from + Trusted Advisor, :py:data:`~awslimitchecker.limit.SOURCE_API` if it + returns a limit retrieved from the service's API, or + :py:data:`~.SOURCE_QUOTAS` if it returns a limit from the Service + Quotas service. + + :returns: one of :py:data:`~awslimitchecker.limit.SOURCE_DEFAULT`, + :py:data:`~awslimitchecker.limit.SOURCE_OVERRIDE`, or + :py:data:`~awslimitchecker.limit.SOURCE_TA`, or + :py:data:`~awslimitchecker.limit.SOURCE_API`, or + :py:data:`~.awslimitchecker.limit.SOURCE_QUOTAS` :rtype: int """ if self.limit_override is not None and ( @@ -185,6 +231,8 @@ def get_limit_source(self): return SOURCE_OVERRIDE if self.api_limit is not None: return SOURCE_API + if self.quotas_limit is not None: + return SOURCE_QUOTAS if self.ta_limit is not None or self.ta_unlimited is True: return SOURCE_TA return SOURCE_DEFAULT @@ -203,6 +251,8 @@ def get_limit(self): return self.limit_override elif limit_type == SOURCE_API: return self.api_limit + elif limit_type == SOURCE_QUOTAS: + return self.quotas_limit elif limit_type == SOURCE_TA: if self.ta_unlimited is True: return None @@ -444,6 +494,40 @@ def ta_limit_name(self): return self._ta_limit_name return self.name + @property + def quotas_service_code(self): + """ + Return the Service Quotas service code to use for this limit. + + :return: Service Quotas service code + :rtype: str + """ + if self._quotas_service_code is not None: + return self._quotas_service_code + return self.service.quotas_service_code + + @property + def quota_name(self): + """ + Return the Service Quotas quota name to use for this limit. + + :return: Service Quotas quota name + :rtype: str + """ + if self._quotas_name is not None: + return self._quotas_name + return self.name + + @property + def quotas_unit(self): + """ + Return the Service Quotas unit to use for this limit. + + :return: Service Quotas unit + :rtype: str + """ + return self._quotas_unit + class AwsLimitUsage(object): diff --git a/awslimitchecker/quotas.py b/awslimitchecker/quotas.py new file mode 100644 index 00000000..5b350b46 --- /dev/null +++ b/awslimitchecker/quotas.py @@ -0,0 +1,154 @@ +""" +awslimitchecker/quotas.py + +The latest version of this package is available at: + + +############################################################################## +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +############################################################################## +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +############################################################################## + +AUTHORS: +Jason Antman +############################################################################## +""" + +from botocore.exceptions import ClientError +import logging + +from awslimitchecker.connectable import Connectable + +logger = logging.getLogger(__name__) + + +class ServiceQuotasClient(Connectable): + api_name = 'service-quotas' + + def __init__(self, boto_connection_kwargs): + """ + Client for the AWS Service Quotas service, that manages retrieving + quotas information and updating :py:class:`~.AwsLimit` instances for + them. This class is also intended to cache Service Quotas responses. + + :param boto_connection_kwargs: keyword arguments to pass to boto3 + connection methods. + :type boto_connection_kwargs: dict + """ + self._boto3_connection_kwargs = boto_connection_kwargs + self._cache = {} + self.conn = None + + def quotas_for_service(self, service_code): + """ + Return this account's current quotas for the specified service code. + Also cache them on this class instance. + + :param service_code: the service code to get quotas for + :type service_code: str + :return: QuotaName to dictionary of quota information returned by the + service + :rtype: dict + """ + if service_code in self._cache: + return self._cache[service_code] + self.connect() + logger.debug( + 'Getting service quotas for service code: %s', service_code + ) + self._cache[service_code] = {} + try: + paginator = self.conn.get_paginator('list_service_quotas') + for page in paginator.paginate(ServiceCode=service_code): + for item in page['Quotas']: + if item['QuotaName'] in self._cache[service_code]: + logger.error( + 'ERROR: Received duplicate service quota for ' + 'service code %s quota name "%s" - QuotaCodes %s' + ' and %s', service_code, item['QuotaName'], + self._cache[service_code][ + item['QuotaName'] + ]['QuotaCode'], item['QuotaCode'] + ) + self._cache[service_code][item['QuotaName'].lower()] = item + except ClientError as ex: + if ex.response.get( + 'Error', {} + ).get('Code', '') == 'NoSuchResourceException': + logger.warning( + 'Attempted to retrieve Service Quotas for service code ' + '%s but received NoSuchResourceException', + service_code + ) + return {} + raise + logger.debug( + 'Retrieved %d quotas for service code %s: %s', + len(self._cache[service_code]), service_code, + sorted([x['QuotaName'] for x in self._cache[service_code].values()]) + ) + return self._cache[service_code] + + def get_quota_value( + self, service_code, quota_name, units='None', converter=None + ): + """ + Return a given quota value, or None if it cannot be found. If + ``units`` is a value other than ``None``, attempt to convert the value + to the specified units. + + :param service_code: the service code to get a quota from + :type service_code: str + :param quota_name: the quota name to get + :type quota_name: str + :param units: the units for the value, or the string "None" + :type units: str + :param converter: A callable for unit conversion. + Must take three positional arguments: the Service Quotas value for + this quota (float), the quota ``Unit`` str, and the return value of + :py:meth:`~.quotas_unit`. This callable is responsible for converting + the quota value from the quota Unit to this class's expected unit. + If they cannot be converted, it should log an error and return None. + :type converter: ``callable`` + :return: the quota value + :rtype: float or None + """ + svc = self.quotas_for_service(service_code) + if quota_name.lower() not in svc: + return None + val = svc[quota_name.lower()].get('Value', None) + if svc[quota_name.lower()]['Unit'] != units: + if converter is not None: + return converter(val, svc[quota_name.lower()]['Unit'], units) + logger.error( + 'ERROR: Service Quota service_code=%s QuotaName="%s" has ' + 'Units set to "%s"; awslimitchecker does not know how to ' + 'handle this. This quota will be ignored. Please open a bug ' + 'report.', service_code, quota_name, + svc[quota_name.lower()]['Unit'] + ) + return None + return val diff --git a/awslimitchecker/runner.py b/awslimitchecker/runner.py index cf705a0a..684553e1 100644 --- a/awslimitchecker/runner.py +++ b/awslimitchecker/runner.py @@ -46,7 +46,7 @@ from .checker import AwsLimitChecker from .utils import StoreKeyValuePair, dict2cols, issue_string_tuple -from .limit import SOURCE_TA, SOURCE_API +from .limit import SOURCE_TA, SOURCE_API, SOURCE_QUOTAS from .metrics import MetricsProvider from .alerts import AlertProvider @@ -185,9 +185,21 @@ def parse_args(self, argv): p.add_argument('-r', '--region', action='store', type=str, default=None, help='AWS region name to connect to; required for STS') + p.add_argument('--role-partition', action='store', type=str, + default='aws', + help='AWS partition name to use for account_role when ' + 'connecting via STS; see documentation for more ' + 'information (default: "aws")') + p.add_argument('--ta-api-region', action='store', type=str, + default='us-east-1', + help='Region to use for Trusted Advisor / Support API' + ' (default: us-east-1)') p.add_argument('--skip-ta', action='store_true', default=False, help='do not attempt to pull *any* information on limits' ' from Trusted Advisor') + p.add_argument('--skip-quotas', action='store_true', default=False, + help='Do not attempt to connect to Service Quotas ' + 'service or use its data for current limits') g = p.add_mutually_exclusive_group() g.add_argument('--ta-refresh-wait', dest='ta_refresh_wait', action='store_true', default=False, @@ -276,6 +288,8 @@ def list_limits(self): src_str = ' (API)' if limits[svc][lim].get_limit_source() == SOURCE_TA: src_str = ' (TA)' + if limits[svc][lim].get_limit_source() == SOURCE_QUOTAS: + src_str = ' (Quotas)' if limits[svc][lim].has_resource_limits(): for usage in limits[svc][lim].get_current_usage(): id = "{s}/{l}/{r}".format(s=svc, l=lim, @@ -427,7 +441,10 @@ def console_entry_point(self): mfa_token=args.mfa_token, ta_refresh_mode=args.ta_refresh_mode, ta_refresh_timeout=args.ta_refresh_timeout, - check_version=args.check_version + check_version=args.check_version, + role_partition=args.role_partition, + ta_api_region=args.ta_api_region, + skip_quotas=args.skip_quotas ) if args.version: diff --git a/awslimitchecker/services/apigateway.py b/awslimitchecker/services/apigateway.py index 6fd78e0d..7e4bfd12 100644 --- a/awslimitchecker/services/apigateway.py +++ b/awslimitchecker/services/apigateway.py @@ -51,6 +51,7 @@ class _ApigatewayService(_AwsService): service_name = 'ApiGateway' api_name = 'apigateway' # AWS API name to connect to (boto3.client) + quotas_service_code = 'apigateway' def find_usage(self): """ @@ -219,7 +220,8 @@ def get_limits(self): 600, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::RestApi' + limit_type='AWS::ApiGateway::RestApi', + quotas_name='Regional APIs' ) limits['Edge APIs per account'] = AwsLimit( 'Edge APIs per account', @@ -227,7 +229,8 @@ def get_limits(self): 120, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::RestApi' + limit_type='AWS::ApiGateway::RestApi', + quotas_name='Edge-optimized APIs' ) limits['Private APIs per account'] = AwsLimit( 'Private APIs per account', @@ -235,7 +238,8 @@ def get_limits(self): 600, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::RestApi' + limit_type='AWS::ApiGateway::RestApi', + quotas_name='Private APIs' ) limits['API keys per account'] = AwsLimit( 'API keys per account', @@ -243,7 +247,8 @@ def get_limits(self): 500, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::ApiKey' + limit_type='AWS::ApiGateway::ApiKey', + quotas_name='API keys' ) limits['Custom authorizers per API'] = AwsLimit( 'Custom authorizers per API', @@ -259,7 +264,8 @@ def get_limits(self): 60, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::ClientCertificate' + limit_type='AWS::ApiGateway::ClientCertificate', + quotas_name='Client certificates' ) limits['Documentation parts per API'] = AwsLimit( 'Documentation parts per API', @@ -275,7 +281,8 @@ def get_limits(self): 300, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::Resource' + limit_type='AWS::ApiGateway::Resource', + quotas_name='Resources/Routes per API' ) limits['Stages per API'] = AwsLimit( 'Stages per API', @@ -283,7 +290,8 @@ def get_limits(self): 10, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::Stage' + limit_type='AWS::ApiGateway::Stage', + quotas_name='Stages per API' ) limits['Usage plans per account'] = AwsLimit( 'Usage plans per account', @@ -291,7 +299,8 @@ def get_limits(self): 300, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::UsagePlan' + limit_type='AWS::ApiGateway::UsagePlan', + quotas_name='Usage plans' ) limits['VPC Links per account'] = AwsLimit( 'VPC Links per account', @@ -299,7 +308,8 @@ def get_limits(self): 5, self.warning_threshold, self.critical_threshold, - limit_type='AWS::ApiGateway::VpcLink' + limit_type='AWS::ApiGateway::VpcLink', + quotas_name='VPC links' ) self.limits = limits return limits diff --git a/awslimitchecker/services/autoscaling.py b/awslimitchecker/services/autoscaling.py index 60b72c6d..7630decc 100644 --- a/awslimitchecker/services/autoscaling.py +++ b/awslimitchecker/services/autoscaling.py @@ -51,6 +51,7 @@ class _AutoscalingService(_AwsService): service_name = 'AutoScaling' api_name = 'autoscaling' + quotas_service_code = 'autoscaling' def find_usage(self): """ @@ -108,6 +109,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::AutoScaling::AutoScalingGroup', + quotas_name='Auto Scaling groups per region' ) # autoscaleconnection.get_all_launch_configurations() limits['Launch configurations'] = AwsLimit( @@ -117,6 +119,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::AutoScaling::LaunchConfiguration', + quotas_name='Launch configurations per region' ) self.limits = limits return limits diff --git a/awslimitchecker/services/base.py b/awslimitchecker/services/base.py index 217c66d3..5a1a715b 100644 --- a/awslimitchecker/services/base.py +++ b/awslimitchecker/services/base.py @@ -47,11 +47,17 @@ class _AwsService(Connectable): __metaclass__ = abc.ABCMeta + #: awslimitchecker's name for the service service_name = 'baseclass' + + #: the AWS API name for the service api_name = 'baseclass' + #: the service code for Service Quotas, or None + quotas_service_code = None + def __init__(self, warning_threshold, critical_threshold, - boto_connection_kwargs={}): + boto_connection_kwargs, quotas_client): """ Describes an AWS service and its limits, and provides methods to query current utilization. @@ -69,37 +75,16 @@ def __init__(self, warning_threshold, critical_threshold, integer percentage, for any limits without a specifically-set threshold. :type critical_threshold: int - :param profile_name: The name of a profile in the cross-SDK - `shared credentials file `_ for boto3 to - retrieve AWS credentials from. - :type profile_name: str - :param account_id: `AWS Account ID `_ - (12-digit string, currently numeric) for the account to connect to - (destination) via STS - :type account_id: str - :param account_role: the name of an - `IAM Role `_ - (in the destination account) to assume - :param region: AWS region name to connect to - :type region: str - :type account_role: str - :param external_id: (optional) the `External ID `_ - string to use when assuming a role via STS. - :type external_id: str - :param mfa_serial_number: (optional) the `MFA Serial Number` string to - use when assuming a role via STS. - :type mfa_serial_number: str - :param mfa_token: (optional) the `MFA Token` string to use when - assuming a role via STS. - :type mfa_token: str + :param boto_connection_kwargs: Dictionary of keyword arguments to + pass to boto connection methods. + :type boto_connection_kwargs: dict + :param quotas_client: Instance of ServiceQuotasClient + :type quotas_client: ``ServiceQuotasClient`` or ``None`` """ self.warning_threshold = warning_threshold self.critical_threshold = critical_threshold self._boto3_connection_kwargs = boto_connection_kwargs + self._quotas_client = quotas_client self.conn = None self.resource_conn = None self.limits = {} @@ -275,3 +260,21 @@ def check_thresholds(self): if limit.check_thresholds() is False: ret[name] = limit return ret + + def _update_service_quotas(self): + """ + Update all limits for this service via the Service Quotas service. + """ + if self.quotas_service_code is None: + return + if self._quotas_client is None: + return + logger.debug('Updating service quotas for %s', self.service_name) + for lname in sorted(self.limits.keys()): + lim = self.limits[lname] + val = self._quotas_client.get_quota_value( + lim.quotas_service_code, lim.quota_name, + units=lim.quotas_unit, converter=lim.quotas_unit_converter + ) + if val is not None: + lim._set_quotas_limit(val) diff --git a/awslimitchecker/services/cloudformation.py b/awslimitchecker/services/cloudformation.py index 83ffe1e5..26e8aa31 100644 --- a/awslimitchecker/services/cloudformation.py +++ b/awslimitchecker/services/cloudformation.py @@ -50,6 +50,7 @@ class _CloudformationService(_AwsService): service_name = 'CloudFormation' api_name = 'cloudformation' # AWS API name to connect to (boto3.client) + quotas_service_code = 'cloudformation' def find_usage(self): """ @@ -95,6 +96,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::CloudFormation::Stack', + quotas_name='Stack count' ) self.limits = limits return limits diff --git a/awslimitchecker/services/cloudtrail.py b/awslimitchecker/services/cloudtrail.py index 6e8e60c4..03e095ac 100644 --- a/awslimitchecker/services/cloudtrail.py +++ b/awslimitchecker/services/cloudtrail.py @@ -71,15 +71,24 @@ def find_usage(self): def _find_usage_cloudtrail(self): """Calculate current usage for CloudTrail related metrics""" - trail_list = self.conn.describe_trails()['trailList'] + trail_list = self.conn.describe_trails( + includeShadowTrails=False + )['trailList'] trail_count = len(trail_list) if trail_list else 0 for trail in trail_list: data_resource_count = 0 if self.conn._client_config.region_name == trail['HomeRegion']: - response = self.conn.get_event_selectors( - TrailName=trail['Name'] - ) + try: + response = self.conn.get_event_selectors( + TrailName=trail['TrailARN'] + ) + except Exception as ex: + logger.debug( + 'Unable to call GetEventSelectors on CloudTrail trail ' + '%s: %s', trail, ex + ) + continue event_selectors = response['EventSelectors'] for event_selector in event_selectors: data_resource_count += len( diff --git a/awslimitchecker/services/dynamodb.py b/awslimitchecker/services/dynamodb.py index d3aa9902..9d8a7da0 100644 --- a/awslimitchecker/services/dynamodb.py +++ b/awslimitchecker/services/dynamodb.py @@ -50,6 +50,7 @@ class _DynamodbService(_AwsService): service_name = 'DynamoDB' api_name = 'dynamodb' + quotas_service_code = 'dynamodb' def find_usage(self): """ @@ -160,7 +161,10 @@ def get_limits(self): 80000 if region_name == 'us-east-1' else 20000, self.warning_threshold, self.critical_threshold, - limit_type='AWS::DynamoDB::Table', ) + limit_type='AWS::DynamoDB::Table', + quotas_name='Account-level write throughput limit ' + '(Provisioned mode)' + ) limits['Table Max Write Capacity Units'] = AwsLimit( 'Table Max Write Capacity Units', @@ -169,6 +173,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::DynamoDB::Table', + quotas_name='Table-level write throughput limit' ) limits['Account Max Read Capacity Units'] = AwsLimit( @@ -178,6 +183,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::DynamoDB::Table', + quotas_name='Account-level read throughput limit (Provisioned mode)' ) limits['Table Max Read Capacity Units'] = AwsLimit( @@ -187,6 +193,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::DynamoDB::Table', + quotas_name='Table-level read throughput limit' ) limits['Global Secondary Indexes'] = AwsLimit( diff --git a/awslimitchecker/services/ebs.py b/awslimitchecker/services/ebs.py index 7a46606f..a6f2444d 100644 --- a/awslimitchecker/services/ebs.py +++ b/awslimitchecker/services/ebs.py @@ -47,10 +47,21 @@ logger = logging.getLogger(__name__) +def convert_TiB_to_GiB(value, in_unit, out_unit): + if in_unit != 'None' or out_unit != 'GiB': + logger.error( + 'ERROR: cannot convert Service Quotas EBS limit value from ' + 'units of "%s" to units of "%s"', in_unit, out_unit + ) + return None + return value * 1024.0 + + class _EbsService(_AwsService): service_name = 'EBS' api_name = 'ec2' + quotas_service_code = 'ebs' def find_usage(self): """ @@ -184,35 +195,45 @@ def _get_limits_ebs(self): self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='io1', + quotas_name='Provisioned IOPS' ) limits['Provisioned IOPS (SSD) storage (GiB)'] = AwsLimit( 'Provisioned IOPS (SSD) storage (GiB)', self, - 102400, + 307200, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='io1', + quotas_name='Provisioned IOPS (SSD) volume storage', + quotas_unit='GiB', + quotas_unit_converter=convert_TiB_to_GiB ) limits['General Purpose (SSD) volume storage (GiB)'] = AwsLimit( 'General Purpose (SSD) volume storage (GiB)', self, - 102400, + 307200, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='gp2', - ta_limit_name='General Purpose SSD (gp2) volume storage (GiB)' + ta_limit_name='General Purpose SSD (gp2) volume storage (GiB)', + quotas_name='General Purpose (SSD) volume storage', + quotas_unit='GiB', + quotas_unit_converter=convert_TiB_to_GiB ) limits['Magnetic volume storage (GiB)'] = AwsLimit( 'Magnetic volume storage (GiB)', self, - 20480, + 307200, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='standard', - ta_limit_name='Magnetic (standard) volume storage (GiB)' + ta_limit_name='Magnetic (standard) volume storage (GiB)', + quotas_name='Magnetic volume storage', + quotas_unit='GiB', + quotas_unit_converter=convert_TiB_to_GiB ) limits['Throughput Optimized (HDD) volume storage (GiB)'] = AwsLimit( 'Throughput Optimized (HDD) volume storage (GiB)', @@ -222,6 +243,9 @@ def _get_limits_ebs(self): self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='st1', + quotas_name='Max Throughput Optimized HDD (ST1) Storage', + quotas_unit='GiB', + quotas_unit_converter=convert_TiB_to_GiB ) limits['Cold (HDD) volume storage (GiB)'] = AwsLimit( 'Cold (HDD) volume storage (GiB)', @@ -231,6 +255,9 @@ def _get_limits_ebs(self): self.critical_threshold, limit_type='AWS::EC2::Volume', limit_subtype='sc1', + quotas_name='Max Cold HDD (SC1) Storage', + quotas_unit='GiB', + quotas_unit_converter=convert_TiB_to_GiB ) limits['Active snapshots'] = AwsLimit( 'Active snapshots', @@ -239,6 +266,7 @@ def _get_limits_ebs(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::VolumeSnapshot', + quotas_name='Number of EBS snapshots' ) limits['Active volumes'] = AwsLimit( 'Active volumes', diff --git a/awslimitchecker/services/ec2.py b/awslimitchecker/services/ec2.py index 06f38563..f1e82fdb 100644 --- a/awslimitchecker/services/ec2.py +++ b/awslimitchecker/services/ec2.py @@ -38,6 +38,7 @@ """ import abc # noqa +import os import logging from collections import defaultdict from copy import deepcopy @@ -56,6 +57,52 @@ class _Ec2Service(_AwsService): service_name = 'EC2' api_name = 'ec2' + quotas_service_code = 'ec2' + + #: Mapping of lower-case instance family character (instance type first + #: character) to limit name for that family. + instance_family_to_limit_name = { + 'f': 'Running On-Demand All F instances', + 'g': 'Running On-Demand All G instances', + 'p': 'Running On-Demand All P instances', + 'x': 'Running On-Demand All X instances' + } + + #: Mapping of lower-case instance family character to Service Quotas + #: quota name for that family. + instance_family_to_quota_name = { + 'f': 'Running On-Demand F instances', + 'g': 'Running On-Demand G instances', + 'p': 'Running On-Demand P instances', + 'x': 'Running On-Demand X instances' + } + + #: Name of default limit for all other (standard) instance families. + default_limit_name = 'Running On-Demand All Standard ' \ + '(A, C, D, H, I, M, R, T, Z) instances' + + #: Name of default Service Quota for all other (standard) families. + default_quota_name = 'Running On-Demand Standard ' \ + '(A, C, D, H, I, M, R, T, Z) instances' + + #: List of instance types that aren't exposed via Service Quotas + no_quotas_types = [ + 'c5d.12xlarge', + 'c5d.24xlarge', + 'c5d.metal', + 'cc1.4xlarge', + 'cg1.4xlarge', + 'cr1.8xlarge', + 'g4dn.metal', + 'hi1.4xlarge', + 'hs1.8xlarge', + 'm5dn.metal', + 'm5n.metal', + 'r5dn.metal', + 'r5n.metal', + 'u-18tb1.metal', + 'u-24tb1.metal' + ] def find_usage(self): """ @@ -68,7 +115,10 @@ def find_usage(self): self.connect_resource() for lim in self.limits.values(): lim._reset_usage() - self._find_usage_instances() + if self._use_vcpu_limits: + self._find_usage_instances_vcpu() + else: + self._find_usage_instances_nonvcpu() self._find_usage_networking_sgs() self._find_usage_networking_eips() self._find_usage_networking_eni_sg() @@ -77,7 +127,7 @@ def find_usage(self): self._have_usage = True logger.debug("Done checking usage.") - def _find_usage_instances(self): + def _find_usage_instances_nonvcpu(self): """calculate On-Demand instance usage for all types and update Limits""" # update our limits with usage inst_usage = self._instance_usage() @@ -128,6 +178,27 @@ def _find_usage_instances(self): aws_type='AWS::EC2::Instance' ) + def _find_usage_instances_vcpu(self): + res_usage = self._get_reserved_instance_count() + logger.debug('Reserved instance count: %s', res_usage) + usage = self._instance_usage_vcpu(res_usage) + limit_values = defaultdict(int) + for i_family, count in usage.items(): + limname = self.instance_family_to_limit_name.get( + i_family, self.default_limit_name + ) + limit_values[limname] += count + for lname in list( + self.instance_family_to_limit_name.values() + ) + [self.default_limit_name]: + if lname not in limit_values: + limit_values[lname] = 0 + for limname, count in limit_values.items(): + self.limits[limname]._add_current_usage( + count, + aws_type='AWS::EC2::Instance', + ) + def _find_usage_spot_instances(self): """calculate spot instance request usage and update Limits""" logger.debug('Getting spot instance request usage') @@ -258,6 +329,75 @@ def _instance_usage(self): "counting", inst.instance_type) return az_to_inst + def _instance_usage_vcpu(self, ris): + """ + Find counts of currently-running EC2 Instance vCPUs + (On-Demand or Reserved) by instance family. Return as a dict of + instance family letter to count. + + :param ris: nested dict of reserved instances, as returned by + :py:meth:`~._get_reserved_instance_count` + :type ris: dict + :rtype: dict + """ + inst_counts = defaultdict(int) + logger.debug("Getting usage for on-demand instances (vCPU limit)") + for inst in self.resource_conn.instances.all(): + if inst.spot_instance_request_id: + logger.info("Spot instance found (%s); skipping from " + "Running On-Demand Instances count", inst.id) + continue + if inst.state['Name'] in ['stopped', 'terminated']: + logger.debug("Ignoring instance %s in state %s", inst.id, + inst.state['Name']) + continue + az = inst.placement['AvailabilityZone'] + itype = inst.instance_type + if ris.get(az, {}).get(itype, 0) > 0: + logger.debug( + 'Using RI for %s: %s in %s', inst.id, itype, az + ) + ris[az][itype] -= 1 + continue + inst_counts[inst.instance_type[0]] += ( + inst.cpu_options['CoreCount'] * inst.cpu_options[ + 'ThreadsPerCore' + ] + ) + return inst_counts + + @property + def _use_vcpu_limits(self): + """ + Return whether or not to use the new vCPU-based limits. + + :return: whether to use vCPU-based limits (True) or older + per-instance-type limits (False) + :rtype: bool + """ + if 'USE_VCPU_LIMITS' in os.environ: + if os.environ['USE_VCPU_LIMITS'] == 'true': + logger.debug( + 'Using vCPU-based EC2 limits due to USE_VCPU_LIMITS=true ' + 'in environment.' + ) + return True + logger.debug( + 'Using vCPU-based EC2 limits due to USE_VCPU_LIMITS in ' + 'environment and set to something other than "true".' + ) + return False + oldconn = self.conn + self.connect() + region_name = self.conn._client_config.region_name + self.conn = oldconn + if region_name.startswith('cn-') or region_name.startswith('us-gov-'): + logger.debug( + 'Using non-vCPU EC2 limits due to region name: %s', region_name + ) + return False + return True + def get_limits(self): """ Return all known limits for this service, as a dict of their names @@ -269,7 +409,10 @@ def get_limits(self): if self.limits != {}: return self.limits limits = {} - limits.update(self._get_limits_instances()) + if self._use_vcpu_limits: + limits.update(self._get_limits_instances_vcpu()) + else: + limits.update(self._get_limits_instances_nonvcpu()) limits.update(self._get_limits_networking()) limits.update(self._get_limits_spot()) self.limits = limits @@ -297,15 +440,16 @@ def _update_limits_from_api(self): lname = 'VPC Elastic IP addresses (EIPs)' elif aname == 'vpc-max-security-groups-per-interface': lname = 'VPC security groups per elastic network interface' - if lname is not None: + if lname in self.limits: if int(val) == 0: continue self.limits[lname]._set_api_limit(int(val)) logger.debug("Done setting limits from API") - def _get_limits_instances(self): + def _get_limits_instances_nonvcpu(self): """ - Return a dict of limits for EC2 instances only. + Return a dict of limits for EC2 instances only, for regions using + non-vCPU-based (old-style) On Demand Instances limits. This method should only be used internally by :py:meth:~.get_limits`. @@ -369,6 +513,9 @@ def _get_limits_instances(self): lim = default_limits[0] if i_type in special_limits: lim = special_limits[i_type][0] + quotas_name = 'Running On-Demand %s instances' % i_type + if i_type in self.no_quotas_types: + quotas_name = None limits[key] = AwsLimit( key, self, @@ -377,7 +524,8 @@ def _get_limits_instances(self): self.critical_threshold, limit_type='On-Demand instances', limit_subtype=i_type, - ta_limit_name='On-Demand instances - %s' % i_type + ta_limit_name='On-Demand instances - %s' % i_type, + quotas_name=quotas_name ) # limit for ALL running On-Demand instances key = 'Running On-Demand EC2 instances' @@ -388,6 +536,41 @@ def _get_limits_instances(self): self.warning_threshold, self.critical_threshold, limit_type='On-Demand instances', + quotas_name='Total running On-Demand instances' + ) + return limits + + def _get_limits_instances_vcpu(self): + """ + Return a dict of limits for EC2 instances only, for regions using + vCPU-based (new-style) On Demand Instances limits. + This method should only be used internally by + :py:meth:~.get_limits`. + + :rtype: dict + """ + limits = {} + iftln = self.instance_family_to_limit_name + for key in iftln.keys(): + limits[iftln[key]] = AwsLimit( + iftln[key], + self, + 128, + self.warning_threshold, + self.critical_threshold, + limit_type='On-Demand instances', + limit_subtype=key.upper(), + quotas_name=self.instance_family_to_quota_name[key] + ) + limits[self.default_limit_name] = AwsLimit( + self.default_limit_name, + self, + 1152, + self.warning_threshold, + self.critical_threshold, + limit_type='On-Demand instances', + limit_subtype='Standard', + quotas_name=self.default_quota_name ) return limits @@ -448,9 +631,38 @@ def _find_usage_networking_sgs(self): sgs_per_vpc = defaultdict(int) rules_per_sg = defaultdict(int) for sg in self.resource_conn.security_groups.all(): - if sg.vpc_id is not None: - sgs_per_vpc[sg.vpc_id] += 1 - rules_per_sg[sg.id] = len(sg.ip_permissions) + if sg.vpc_id is None: + continue + sgs_per_vpc[sg.vpc_id] += 1 + """ + see: https://github.com/jantman/awslimitchecker/issues/431 + + The value for each of ingress and egress is the count of all + PrefixListIds in all rules, plus the count of all + UserIdGroupPairs in all rules, plus the maximum of: + the count of all IpRanges in all rules + -or- + the count of all Ipv6Ranges in all rules + + The limit that we alert on is the maximum of those values for + ingress and egress. + + In short, behind the scenes, there are four firewall rulesets + per SG: (IPv4|IPv6) (ingress|egress) + Each can have a maximum of entries. PrefixListIds and + UserIdGroupPairs count towards both IPv4 and IPv6. + """ + counts = [] + for perm in [sg.ip_permissions, sg.ip_permissions_egress]: + counts.append( + max( + sum([len(x.get('IpRanges', [])) for x in perm]), + sum([len(x.get('Ipv6Ranges', [])) for x in perm]) + ) + + sum([len(x.get('PrefixListIds', [])) for x in perm]) + + sum([len(x.get('UserIdGroupPairs', [])) for x in perm]) + ) + rules_per_sg[sg.id] = max(counts) # set usage for vpc_id, count in sgs_per_vpc.items(): self.limits['Security groups per VPC']._add_current_usage( @@ -528,7 +740,8 @@ def _get_limits_networking(self): self.critical_threshold, limit_type='AWS::EC2::EIP', limit_subtype='AWS::EC2::VPC', - ta_service_name='VPC' # TA shows this as VPC not EC2 + ta_service_name='VPC', # TA shows this as VPC not EC2 + quotas_name='Number of EIPs - VPC EIPs' ) # the EC2 limits screen calls this 'EC2-Classic Elastic IPs' # but Trusted Advisor just calls it 'Elastic IP addresses (EIPs)' @@ -539,6 +752,7 @@ def _get_limits_networking(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::EIP', + quotas_name='Elastic IP addresses for EC2-Classic' ) limits['VPC security groups per elastic network interface'] = AwsLimit( 'VPC security groups per elastic network interface', @@ -594,49 +808,91 @@ def _instance_types(self): 'a1.4xlarge', 'a1.large', 'a1.medium', + 'a1.metal', 'a1.xlarge', - 't2.nano', - 't2.micro', - 't2.small', - 't2.medium', - 't2.large', - 't2.xlarge', - 't2.2xlarge', - 't3.nano', - 't3.micro', - 't3.small', - 't3.medium', - 't3.large', - 't3.xlarge', - 't3.2xlarge', - 'm3.medium', + 'm3.2xlarge', 'm3.large', + 'm3.medium', 'm3.xlarge', - 'm3.2xlarge', - 'm4.large', - 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge', 'm4.16xlarge', - 'm5.12xlarge', - 'm5.24xlarge', + 'm4.large', + 'm4.xlarge', 'm5.2xlarge', 'm5.4xlarge', + 'm5.8xlarge', + 'm5.12xlarge', + 'm5.16xlarge', + 'm5.24xlarge', 'm5.large', + 'm5.metal', 'm5.xlarge', - 'm5d.12xlarge', - 'm5d.24xlarge', - 'm5d.2xlarge', - 'm5d.4xlarge', - 'm5d.large', - 'm5d.xlarge', - 'm5a.12xlarge', - 'm5a.24xlarge', 'm5a.2xlarge', 'm5a.4xlarge', + 'm5a.8xlarge', + 'm5a.12xlarge', + 'm5a.16xlarge', + 'm5a.24xlarge', 'm5a.large', 'm5a.xlarge', + 'm5ad.2xlarge', + 'm5ad.4xlarge', + 'm5ad.8xlarge', + 'm5ad.12xlarge', + 'm5ad.16xlarge', + 'm5ad.24xlarge', + 'm5ad.large', + 'm5ad.xlarge', + 'm5d.2xlarge', + 'm5d.4xlarge', + 'm5d.8xlarge', + 'm5d.12xlarge', + 'm5d.16xlarge', + 'm5d.24xlarge', + 'm5d.large', + 'm5d.metal', + 'm5d.xlarge', + 'm5dn.2xlarge', + 'm5dn.4xlarge', + 'm5dn.8xlarge', + 'm5dn.12xlarge', + 'm5dn.16xlarge', + 'm5dn.24xlarge', + 'm5dn.large', + 'm5dn.metal', + 'm5dn.xlarge', + 'm5n.2xlarge', + 'm5n.4xlarge', + 'm5n.8xlarge', + 'm5n.12xlarge', + 'm5n.16xlarge', + 'm5n.24xlarge', + 'm5n.large', + 'm5n.metal', + 'm5n.xlarge', + 't2.2xlarge', + 't2.large', + 't2.medium', + 't2.micro', + 't2.nano', + 't2.small', + 't2.xlarge', + 't3.2xlarge', + 't3.large', + 't3.medium', + 't3.micro', + 't3.nano', + 't3.small', + 't3.xlarge', + 't3a.2xlarge', + 't3a.large', + 't3a.medium', + 't3a.micro', + 't3a.nano', + 't3a.small', + 't3a.xlarge', ] PREV_GENERAL_TYPES = [ @@ -668,12 +924,22 @@ def _instance_types(self): 'r5.large', 'r5.metal', 'r5.xlarge', - 'r5a.12xlarge', - 'r5a.24xlarge', 'r5a.2xlarge', 'r5a.4xlarge', + 'r5a.8xlarge', + 'r5a.12xlarge', + 'r5a.16xlarge', + 'r5a.24xlarge', 'r5a.large', 'r5a.xlarge', + 'r5ad.2xlarge', + 'r5ad.4xlarge', + 'r5ad.8xlarge', + 'r5ad.12xlarge', + 'r5ad.16xlarge', + 'r5ad.24xlarge', + 'r5ad.large', + 'r5ad.xlarge', 'r5d.2xlarge', 'r5d.4xlarge', 'r5d.8xlarge', @@ -683,6 +949,26 @@ def _instance_types(self): 'r5d.large', 'r5d.metal', 'r5d.xlarge', + 'r5dn.2xlarge', + 'r5dn.4xlarge', + 'r5dn.8xlarge', + 'r5dn.12xlarge', + 'r5dn.16xlarge', + 'r5dn.24xlarge', + 'r5dn.large', + 'r5dn.metal', + 'r5dn.xlarge', + 'r5n.2xlarge', + 'r5n.4xlarge', + 'r5n.8xlarge', + 'r5n.12xlarge', + 'r5n.16xlarge', + 'r5n.24xlarge', + 'r5n.large', + 'r5n.metal', + 'r5n.xlarge', + 'u-18tb1.metal', + 'u-24tb1.metal', 'x1.16xlarge', 'x1.32xlarge', 'x1e.2xlarge', @@ -707,33 +993,40 @@ def _instance_types(self): ] COMPUTE_TYPES = [ - 'c3.large', - 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge', 'c3.8xlarge', - 'c4.large', - 'c4.xlarge', + 'c3.large', + 'c3.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', - 'c5.18xlarge', + 'c4.large', + 'c4.xlarge', 'c5.2xlarge', 'c5.4xlarge', 'c5.9xlarge', + 'c5.12xlarge', + 'c5.18xlarge', + 'c5.24xlarge', 'c5.large', + 'c5.metal', 'c5.xlarge', - 'c5d.18xlarge', 'c5d.2xlarge', 'c5d.4xlarge', 'c5d.9xlarge', + 'c5d.12xlarge', + 'c5d.18xlarge', + 'c5d.24xlarge', 'c5d.large', + 'c5d.metal', 'c5d.xlarge', - 'c5n.18xlarge', 'c5n.2xlarge', 'c5n.4xlarge', 'c5n.9xlarge', + 'c5n.18xlarge', 'c5n.large', + 'c5n.metal', 'c5n.xlarge', ] @@ -756,21 +1049,28 @@ def _instance_types(self): ] STORAGE_TYPES = [ - 'i2.xlarge', + 'h1.2xlarge', + 'h1.4xlarge', + 'h1.8xlarge', + 'h1.16xlarge', 'i2.2xlarge', 'i2.4xlarge', 'i2.8xlarge', - 'i3.large', - 'i3.xlarge', + 'i2.xlarge', 'i3.2xlarge', 'i3.4xlarge', 'i3.8xlarge', 'i3.16xlarge', + 'i3.large', 'i3.metal', - 'h1.16xlarge', - 'h1.2xlarge', - 'h1.4xlarge', - 'h1.8xlarge', + 'i3.xlarge', + 'i3en.2xlarge', + 'i3en.3xlarge', + 'i3en.6xlarge', + 'i3en.12xlarge', + 'i3en.24xlarge', + 'i3en.large', + 'i3en.xlarge', ] PREV_STORAGE_TYPES = [ @@ -790,10 +1090,17 @@ def _instance_types(self): GPU_TYPES = [ 'g2.2xlarge', 'g2.8xlarge', - 'g3.16xlarge', 'g3.4xlarge', 'g3.8xlarge', + 'g3.16xlarge', 'g3s.xlarge', + 'g4dn.2xlarge', + 'g4dn.4xlarge', + 'g4dn.8xlarge', + 'g4dn.12xlarge', + 'g4dn.16xlarge', + 'g4dn.metal', + 'g4dn.xlarge', ] PREV_GPU_TYPES = [ diff --git a/awslimitchecker/services/efs.py b/awslimitchecker/services/efs.py index e38f13a7..c0e73a77 100644 --- a/awslimitchecker/services/efs.py +++ b/awslimitchecker/services/efs.py @@ -52,6 +52,7 @@ class _EfsService(_AwsService): service_name = 'EFS' api_name = 'efs' # AWS API name to connect to (boto3.client) + quotas_service_code = 'elasticfilesystem' def find_usage(self): """ @@ -91,10 +92,6 @@ def get_limits(self): Return all known limits for this service, as a dict of their names to :py:class:`~.AwsLimit` objects. - **Note:** we can't make connections to AWS in this method. So, the - :py:meth:`~._update_limits_from_api` method fixes this limit if we're - in us-east-1, which has a lower default limit. - :returns: dict of limit names to :py:class:`~.AwsLimit` objects :rtype: dict """ @@ -108,27 +105,11 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::EFS::FileSystem', + quotas_name='File systems per account' ) self.limits = limits return limits - def _update_limits_from_api(self): - """ - Call :py:meth:`~.connect` and then check what region we're running in; - adjust default limits as required for regions that differ (us-east-1). - """ - region_limits = { - 'us-east-1': 70 - } - self.connect() - rname = self.conn._client_config.region_name - if rname in region_limits: - self.limits['File systems'].default_limit = region_limits[rname] - logger.debug( - 'Running in region %s; setting EFS "File systems" default ' - 'limit value to: %d', rname, region_limits[rname] - ) - def required_iam_permissions(self): """ Return a list of IAM Actions required for this Service to function diff --git a/awslimitchecker/services/elb.py b/awslimitchecker/services/elb.py index ffde8aa5..635e9430 100644 --- a/awslimitchecker/services/elb.py +++ b/awslimitchecker/services/elb.py @@ -61,6 +61,7 @@ class _ElbService(_AwsService): service_name = 'ELB' api_name = 'elb' + quotas_service_code = 'elasticloadbalancing' def find_usage(self): """ @@ -271,6 +272,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::ElasticLoadBalancing::LoadBalancer', + quotas_name='Classic Load Balancers per Region', + quotas_unit='Count' ) limits['Listeners per load balancer'] = AwsLimit( 'Listeners per load balancer', @@ -298,6 +301,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::ElasticLoadBalancingV2::LoadBalancer', + quotas_name='Application Load Balancers per Region', + quotas_unit='Count' ) limits['Target groups'] = AwsLimit( 'Target groups', diff --git a/awslimitchecker/services/firehose.py b/awslimitchecker/services/firehose.py index 764512dc..686f3352 100644 --- a/awslimitchecker/services/firehose.py +++ b/awslimitchecker/services/firehose.py @@ -51,6 +51,7 @@ class _FirehoseService(_AwsService): service_name = 'Firehose' api_name = 'firehose' + quotas_service_code = 'firehose' def find_usage(self): """ @@ -106,6 +107,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::KinesisFirehose::DeliveryStream', + quotas_name='Delivery streams' ) self.limits = limits return limits diff --git a/awslimitchecker/services/iam.py b/awslimitchecker/services/iam.py index 1f426464..0076d533 100644 --- a/awslimitchecker/services/iam.py +++ b/awslimitchecker/services/iam.py @@ -50,6 +50,7 @@ class _IamService(_AwsService): service_name = 'IAM' api_name = 'iam' + quotas_service_code = 'iam' # mapping of iam.AccountSummary() key to limit name API_TO_LIMIT_NAME = { @@ -93,6 +94,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::Group', + quotas_name='Groups per account' ) limits['Users'] = AwsLimit( 'Users', @@ -101,6 +103,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::User', + quotas_name='Users per account' ) limits['Roles'] = AwsLimit( 'Roles', @@ -109,6 +112,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::Role', + quotas_name='Roles per account' ) limits['Instance profiles'] = AwsLimit( 'Instance profiles', @@ -117,6 +121,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::InstanceProfile', + quotas_name='Instance profiles per account' ) limits['Server certificates'] = AwsLimit( 'Server certificates', @@ -125,6 +130,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::ServerCertificate', + quotas_name='Server certificates per account' ) limits['Policies'] = AwsLimit( 'Policies', @@ -133,6 +139,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::IAM::Policy', + quotas_name='Customer managed policies per account' ) limits['Policy Versions In Use'] = AwsLimit( 'Policy Versions In Use', diff --git a/awslimitchecker/services/newservice.py.example b/awslimitchecker/services/newservice.py.example index 4c526e3b..48d236e1 100644 --- a/awslimitchecker/services/newservice.py.example +++ b/awslimitchecker/services/newservice.py.example @@ -50,6 +50,7 @@ class _XXNewServiceXXService(_AwsService): service_name = 'XXNewServiceXX' api_name = 'XXnewserviceXX' # AWS API name to connect to (boto3.client) + quotas_service_code = None # @TODO set to Service Quotas service code, if available def find_usage(self): """ diff --git a/awslimitchecker/services/rds.py b/awslimitchecker/services/rds.py index 1f37da54..2c0080a2 100644 --- a/awslimitchecker/services/rds.py +++ b/awslimitchecker/services/rds.py @@ -50,6 +50,7 @@ class _RDSService(_AwsService): service_name = 'RDS' api_name = 'rds' + quotas_service_code = 'rds' # Mapping of RDS DescribeAccountAttributes action AccountQuotaName string # to our Limit name @@ -147,7 +148,8 @@ def get_limits(self): 40, self.warning_threshold, self.critical_threshold, - limit_type='AWS::RDS::DBInstance' + limit_type='AWS::RDS::DBInstance', + quotas_name='DB instances' ) limits['Reserved Instances'] = AwsLimit( 'Reserved Instances', @@ -156,6 +158,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBInstance', + quotas_name='Reserved DB instances' ) limits['Storage quota (GB)'] = AwsLimit( 'Storage quota (GB)', @@ -164,6 +167,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBInstance', + quotas_name='Total storage for all DB instances', + quotas_unit='Gigabytes' ) limits['DB snapshots per user'] = AwsLimit( 'DB snapshots per user', @@ -172,6 +177,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBSnapshot', + quotas_name='Manual snapshots' ) limits['DB parameter groups'] = AwsLimit( 'DB parameter groups', @@ -180,6 +186,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBParameterGroup', + quotas_name='Parameter groups' ) limits['DB security groups'] = AwsLimit( 'DB security groups', @@ -188,6 +195,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBSecurityGroup', + quotas_name='Security groups' ) limits['VPC Security Groups'] = AwsLimit( 'VPC Security Groups', @@ -204,7 +212,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBSubnetGroup', - ta_limit_name='Subnet groups' + ta_limit_name='Subnet groups', + quotas_name='DB subnet groups' ) limits['Subnets per Subnet Group'] = AwsLimit( 'Subnets per Subnet Group', @@ -213,7 +222,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBSubnetGroup', - ta_limit_name='Subnets per subnet group' + ta_limit_name='Subnets per subnet group', + quotas_name='Subnets per DB subnet group' ) limits['Option Groups'] = AwsLimit( 'Option Groups', @@ -222,6 +232,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBOptionGroup', + quotas_name='Option groups' ) limits['Event Subscriptions'] = AwsLimit( 'Event Subscriptions', @@ -230,7 +241,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBEventSubscription', - ta_limit_name='Event subscriptions' + ta_limit_name='Event subscriptions', + quotas_name='Event subscriptions' ) limits['Read replicas per master'] = AwsLimit( 'Read replicas per master', @@ -239,6 +251,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBInstance', + quotas_name='Read replicas per master' ) # this is the number of rules per security group limits['Max auths per security group'] = AwsLimit( @@ -249,6 +262,7 @@ def get_limits(self): self.critical_threshold, limit_type='AWS::RDS::DBSecurityGroup', limit_subtype='AWS::RDS::DBSecurityGroupIngress', + quotas_name='Authorizations per DB security group' ) limits['DB Clusters'] = AwsLimit( 'DB Clusters', @@ -257,7 +271,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBCluster', - ta_limit_name='Clusters' + ta_limit_name='Clusters', + quotas_name='DB clusters' ) limits['DB Cluster Parameter Groups'] = AwsLimit( 'DB Cluster Parameter Groups', @@ -266,7 +281,8 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::RDS::DBClusterParameterGroup', - ta_limit_name='Cluster parameter groups' + ta_limit_name='Cluster parameter groups', + quotas_name='DB cluster parameter groups' ) self.limits = limits return limits diff --git a/awslimitchecker/services/vpc.py b/awslimitchecker/services/vpc.py index 500cb8b7..477560f5 100644 --- a/awslimitchecker/services/vpc.py +++ b/awslimitchecker/services/vpc.py @@ -55,6 +55,7 @@ class _VpcService(_AwsService): service_name = 'VPC' api_name = 'ec2' + quotas_service_code = 'vpc' def find_usage(self): """ @@ -246,6 +247,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::VPC', + quotas_name='VPCs per Region' ) limits['Subnets per VPC'] = AwsLimit( @@ -305,6 +307,7 @@ def get_limits(self): self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::InternetGateway', + quotas_name='Internet gateways per Region' ) limits['NAT Gateways per AZ'] = AwsLimit( diff --git a/awslimitchecker/tests/services/result_fixtures.py b/awslimitchecker/tests/services/result_fixtures.py index 0802b5ea..e1b25b91 100644 --- a/awslimitchecker/tests/services/result_fixtures.py +++ b/awslimitchecker/tests/services/result_fixtures.py @@ -1904,6 +1904,167 @@ def test_instance_usage(self): ] return return_value + @property + def test_instance_usage_vcpu(self): + mock_inst1A = Mock(spec_set=Instance) + type(mock_inst1A).id = '1A' + type(mock_inst1A).instance_type = 't2.micro' + type(mock_inst1A).spot_instance_request_id = None + type(mock_inst1A).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst1A).state = {'Code': 16, 'Name': 'running'} + type(mock_inst1A).cpu_options = {'CoreCount': 1, 'ThreadsPerCore': 2} + + mock_inst1B = Mock(spec_set=Instance) + type(mock_inst1B).id = '1B' + type(mock_inst1B).instance_type = 'r3.2xlarge' + type(mock_inst1B).spot_instance_request_id = None + type(mock_inst1B).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst1B).state = {'Code': 0, 'Name': 'pending'} + type(mock_inst1B).cpu_options = {'CoreCount': 4, 'ThreadsPerCore': 2} + + mock_inst2A = Mock(spec_set=Instance) + type(mock_inst2A).id = '2A' + type(mock_inst2A).instance_type = 'c4.4xlarge' + type(mock_inst2A).spot_instance_request_id = None + type(mock_inst2A).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2A).state = {'Code': 32, 'Name': 'shutting-down'} + type(mock_inst2A).cpu_options = {'CoreCount': 8, 'ThreadsPerCore': 2} + + mock_inst2B = Mock(spec_set=Instance) + type(mock_inst2B).id = '2B' + type(mock_inst2B).instance_type = 't2.micro' + type(mock_inst2B).spot_instance_request_id = '1234' + type(mock_inst2B).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2B).state = {'Code': 64, 'Name': 'stopping'} + type(mock_inst2B).cpu_options = {'CoreCount': 1, 'ThreadsPerCore': 2} + + mock_inst2C = Mock(spec_set=Instance) + type(mock_inst2C).id = '2C' + type(mock_inst2C).instance_type = 'm4.8xlarge' + type(mock_inst2C).spot_instance_request_id = None + type(mock_inst2C).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2C).state = {'Code': 16, 'Name': 'running'} + type(mock_inst2C).cpu_options = {'CoreCount': 16, 'ThreadsPerCore': 2} + + mock_instStopped = Mock(spec_set=Instance) + type(mock_instStopped).id = 'instStopped' + type(mock_instStopped).instance_type = 'm4.8xlarge' + type(mock_instStopped).spot_instance_request_id = None + type(mock_instStopped).placement = {'AvailabilityZone': 'az1a'} + type(mock_instStopped).state = {'Code': 80, 'Name': 'stopped'} + type(mock_instStopped).cpu_options = { + 'CoreCount': 16, 'ThreadsPerCore': 2 + } + + mock_instTerm = Mock(spec_set=Instance) + type(mock_instTerm).id = '2C' + type(mock_instTerm).instance_type = 'm4.8xlarge' + type(mock_instTerm).spot_instance_request_id = None + type(mock_instTerm).placement = {'AvailabilityZone': 'az1a'} + type(mock_instTerm).state = {'Code': 48, 'Name': 'terminated'} + type(mock_instTerm).cpu_options = {'CoreCount': 16, 'ThreadsPerCore': 2} + + mock_inst2D = Mock(spec_set=Instance) + type(mock_inst2D).id = '2D' + type(mock_inst2D).instance_type = 'f1.16xlarge' + type(mock_inst2D).spot_instance_request_id = None + type(mock_inst2D).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2D).state = {'Code': 16, 'Name': 'running'} + type(mock_inst2D).cpu_options = {'CoreCount': 32, 'ThreadsPerCore': 2} + + mock_inst2E = Mock(spec_set=Instance) + type(mock_inst2E).id = '2E' + type(mock_inst2E).instance_type = 'f1.2xlarge' + type(mock_inst2E).spot_instance_request_id = None + type(mock_inst2E).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2E).state = {'Code': 16, 'Name': 'running'} + type(mock_inst2E).cpu_options = {'CoreCount': 4, 'ThreadsPerCore': 2} + + mock_inst2F = Mock(spec_set=Instance) + type(mock_inst2F).id = '2F' + type(mock_inst2F).instance_type = 'g4dn.12xlarge' + type(mock_inst2F).spot_instance_request_id = None + type(mock_inst2F).placement = {'AvailabilityZone': 'az1a'} + type(mock_inst2F).state = {'Code': 16, 'Name': 'running'} + type(mock_inst2F).cpu_options = {'CoreCount': 12, 'ThreadsPerCore': 4} + + mock_inst3A = Mock(spec_set=Instance) + type(mock_inst1A).id = '3A' + type(mock_inst3A).instance_type = 'p2.16xlarge' + type(mock_inst3A).spot_instance_request_id = None + type(mock_inst3A).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3A).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3A).cpu_options = {'CoreCount': 32, 'ThreadsPerCore': 2} + + mock_inst3F = Mock(spec_set=Instance) + type(mock_inst3F).id = '3F' + type(mock_inst3F).instance_type = 'p2.8xlarge' + type(mock_inst3F).spot_instance_request_id = None + type(mock_inst3F).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3F).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3F).cpu_options = {'CoreCount': 16, 'ThreadsPerCore': 2} + + mock_inst3G = Mock(spec_set=Instance) + type(mock_inst3G).id = '3G' + type(mock_inst3G).instance_type = 'p2.8xlarge' + type(mock_inst3G).spot_instance_request_id = None + type(mock_inst3G).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3G).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3G).cpu_options = {'CoreCount': 16, 'ThreadsPerCore': 2} + + mock_inst3B = Mock(spec_set=Instance) + type(mock_inst3B).id = '3B' + type(mock_inst3B).instance_type = 'r3.2xlarge' + type(mock_inst3B).spot_instance_request_id = None + type(mock_inst3B).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3B).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3B).cpu_options = {'CoreCount': 4, 'ThreadsPerCore': 2} + + mock_inst3C = Mock(spec_set=Instance) + type(mock_inst3C).id = '3C' + type(mock_inst3C).instance_type = 'x1e.32xlarge' + type(mock_inst3C).spot_instance_request_id = None + type(mock_inst3C).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3C).state = {'Code': 32, 'Name': 'stopped'} + type(mock_inst3C).cpu_options = {'CoreCount': 32, 'ThreadsPerCore': 4} + + mock_inst3D = Mock(spec_set=Instance) + type(mock_inst3D).id = '3D' + type(mock_inst3D).instance_type = 'x1e.32xlarge' + type(mock_inst3D).spot_instance_request_id = None + type(mock_inst3D).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3D).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3D).cpu_options = {'CoreCount': 32, 'ThreadsPerCore': 4} + + mock_inst3E = Mock(spec_set=Instance) + type(mock_inst3E).id = '3E' + type(mock_inst3E).instance_type = 'x1e.32xlarge' + type(mock_inst3E).spot_instance_request_id = None + type(mock_inst3E).placement = {'AvailabilityZone': 'az1c'} + type(mock_inst3E).state = {'Code': 16, 'Name': 'running'} + type(mock_inst3E).cpu_options = {'CoreCount': 32, 'ThreadsPerCore': 4} + + return_value = [ + mock_inst1A, + mock_inst1B, + mock_inst2A, + mock_inst2B, + mock_inst2C, + mock_instStopped, + mock_instTerm, + mock_inst2D, + mock_inst2E, + mock_inst2F, + mock_inst3A, + mock_inst3B, + mock_inst3C, + mock_inst3D, + mock_inst3E, + mock_inst3F, + mock_inst3G, + ] + return return_value + @property def test_instance_usage_key_error(self): mock_inst1A = Mock(spec_set=Instance) @@ -1923,19 +2084,458 @@ def test_find_usage_networking_sgs(self): type(mock_sg1).ip_permissions_egress = [] mock_sg2 = Mock(spec_set=SecurityGroup) type(mock_sg2).id = 'sg-2' - type(mock_sg2).vpc_id = None - type(mock_sg2).ip_permissions = [1, 2, 3, 4, 5, 6] - type(mock_sg2).ip_permissions_egress = [8, 9, 10] + type(mock_sg2).vpc_id = 'vpc-aaa' + type(mock_sg2).ip_permissions = [ + { + 'FromPort': 1, + 'ToPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + {1: 1}, {2: 2}, {3: 3}, {4: 4} + ], + 'Ipv6Ranges': [ + {1: 1}, {2: 2} + ], + 'PrefixListIds': [ + {'p1': 'p1'}, + ], + 'UserIdGroupPairs': [ + {'a': 'a'}, {'b': 'b'} + ] + }, + { + 'FromPort': 2, + 'IpProtocol': 'string', + 'IpRanges': [ + {1: 1}, + ], + 'Ipv6Ranges': [], + 'PrefixListIds': [], + 'ToPort': 123, + 'UserIdGroupPairs': [] + }, + { + 'FromPort': 3, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [ + {1: 1}, + ], + 'PrefixListIds': [], + 'ToPort': 123, + 'UserIdGroupPairs': [ + {'a': 'a'}, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [], + 'PrefixListIds': [ + {'a': 'a'}, + ], + 'ToPort': 1, + 'UserIdGroupPairs': [ + {'b': 'b'}, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + {'c': 'c'}, + ], + 'Ipv6Ranges': [ + {4: 4}, + ], + 'PrefixListIds': [ + {5: 5}, {6: 6} + ], + 'ToPort': 2, + 'UserIdGroupPairs': [] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [], + 'PrefixListIds': [ + {2: 2}, + ], + 'ToPort': 3, + 'UserIdGroupPairs': [] + } + ] + type(mock_sg2).ip_permissions_egress = [ + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + {1: 1}, + ], + 'Ipv6Ranges': [ + {2: 2}, {3: 3}, {4: 4} + ], + 'PrefixListIds': [ + {5: 5}, + ], + 'ToPort': 1, + 'UserIdGroupPairs': [ + {6: 6}, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [ + {7: 7}, + ], + 'PrefixListIds': [], + 'ToPort': 2, + 'UserIdGroupPairs': [] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [ + {8: 8}, + ], + 'PrefixListIds': [], + 'ToPort': 3, + 'UserIdGroupPairs': [] + } + ] mock_sg3 = Mock(spec_set=SecurityGroup) type(mock_sg3).id = 'sg-3' type(mock_sg3).vpc_id = 'vpc-bbb' - type(mock_sg3).ip_permissions = [1, 2, 3, 4, 5, 6, 7, 8, 9] - type(mock_sg3).ip_permissions_egress = [6, 7, 8, 9] + type(mock_sg3).ip_permissions = [ + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + {1: 1}, + ], + 'Ipv6Ranges': [], + 'PrefixListIds': [ + {'a': 'a'}, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + {2: 2}, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [], + 'Ipv6Ranges': [ + {3: 3}, {6: 6} + ], + 'PrefixListIds': [ + {4: 4}, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [] + } + ] + type(mock_sg3).ip_permissions_egress = [ + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + {1: 1}, + {2: 2}, + {3: 3}, + {4: 4}, + {5: 5}, + {6: 6}, + ], + 'Ipv6Ranges': [ + {1: 1}, + {2: 2}, + {3: 3}, + {4: 4}, + {5: 5}, + {6: 6}, + {7: 7}, + {8: 8}, + {9: 9}, + {10: 10}, + {11: 11}, + {12: 12}, + {13: 13} + ], + 'PrefixListIds': [ + {1: 1}, + {2: 2}, + {3: 3}, + {4: 4}, + {5: 5}, + {6: 6}, + {7: 7}, + {8: 8}, + {9: 9}, + {10: 10} + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + {1: 1}, + {2: 2}, + {3: 3}, + {4: 4}, + {5: 5}, + {6: 6}, + ] + } + ] mock_sg4 = Mock(spec_set=SecurityGroup) type(mock_sg4).id = 'sg-4' - type(mock_sg4).vpc_id = 'vpc-aaa' - type(mock_sg4).ip_permissions = [1, 2, 3] - type(mock_sg4).ip_permissions_egress = [21, 22, 23, 24] + type(mock_sg4).vpc_id = None + type(mock_sg4).ip_permissions = [ + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + ] + type(mock_sg4).ip_permissions_egress = [ + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + }, + { + 'FromPort': 123, + 'IpProtocol': 'string', + 'IpRanges': [ + { + 'CidrIp': 'string', + 'Description': 'string' + }, + ], + 'Ipv6Ranges': [ + { + 'CidrIpv6': 'string', + 'Description': 'string' + }, + ], + 'PrefixListIds': [ + { + 'Description': 'string', + 'PrefixListId': 'string' + }, + ], + 'ToPort': 123, + 'UserIdGroupPairs': [ + { + 'Description': 'string', + 'GroupId': 'string', + 'GroupName': 'string', + 'PeeringStatus': 'string', + 'UserId': 'string', + 'VpcId': 'string', + 'VpcPeeringConnectionId': 'string' + }, + ] + } + ] return_value = [ mock_sg1, @@ -2107,6 +2707,58 @@ def test_find_usage_networking_eni_sg(self): ] } + test_update_limits_from_api_vcpu = { + 'ResponseMetadata': { + 'HTTPStatusCode': 200, + 'RequestId': '16b85906-ab0d-4134-b8bb-df3e6120c6c7' + }, + 'AccountAttributes': [ + { + 'AttributeName': 'supported-platforms', + 'AttributeValues': [ + { + 'AttributeValue': 'EC2' + }, + { + 'AttributeValue': 'VPC' + } + ] + }, + { + 'AttributeName': 'vpc-max-security-groups-per-interface', + 'AttributeValues': [ + { + 'AttributeValue': '5' + } + ] + }, + { + 'AttributeName': 'max-elastic-ips', + 'AttributeValues': [ + { + 'AttributeValue': '40' + } + ] + }, + { + 'AttributeName': 'vpc-max-elastic-ips', + 'AttributeValues': [ + { + 'AttributeValue': '200' + } + ] + }, + { + 'AttributeName': 'default-vpc', + 'AttributeValues': [ + { + 'AttributeValue': 'none' + } + ] + } + ] + } + test_update_limits_from_api_unsupported = { 'ResponseMetadata': { 'HTTPStatusCode': 200, @@ -3736,7 +4388,7 @@ class CloudTrail(object): 'IncludeGlobalServiceEvents': True, 'IsMultiRegionTrail': True, 'HomeRegion': 'thisregion', - 'TrailARN': 'string', + 'TrailARN': 'trailarn1', 'LogFileValidationEnabled': True, 'CloudWatchLogsLogGroupArn': 'string', 'CloudWatchLogsRoleArn': 'string', @@ -3752,7 +4404,7 @@ class CloudTrail(object): 'IncludeGlobalServiceEvents': True, 'IsMultiRegionTrail': True, 'HomeRegion': 'thisregion', - 'TrailARN': 'string', + 'TrailARN': 'trailarn2', 'LogFileValidationEnabled': True, 'CloudWatchLogsLogGroupArn': 'string', 'CloudWatchLogsRoleArn': 'string', @@ -3768,12 +4420,17 @@ class CloudTrail(object): 'IncludeGlobalServiceEvents': True, 'IsMultiRegionTrail': True, 'HomeRegion': 'otherRegion', - 'TrailARN': 'string', + 'TrailARN': 'trailarn3', 'LogFileValidationEnabled': True, 'CloudWatchLogsLogGroupArn': 'string', 'CloudWatchLogsRoleArn': 'string', 'KmsKeyId': 'string', 'HasCustomEventSelectors': True + }, + { + 'Name': 'trail4', + 'TrailARN': 'trailarn4', + 'HomeRegion': 'thisregion' } ], } diff --git a/awslimitchecker/tests/services/test_apigateway.py b/awslimitchecker/tests/services/test_apigateway.py index 52c6c723..a0cda022 100644 --- a/awslimitchecker/tests/services/test_apigateway.py +++ b/awslimitchecker/tests/services/test_apigateway.py @@ -61,15 +61,16 @@ class Test_ApigatewayService(object): def test_init(self): """test __init__()""" - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) assert cls.service_name == 'ApiGateway' assert cls.api_name == 'apigateway' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 + assert cls.quotas_service_code == 'apigateway' def test_get_limits(self): - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -89,11 +90,12 @@ def test_get_limits(self): assert limit.service == cls assert limit.def_warning_threshold == 21 assert limit.def_critical_threshold == 43 + assert res['VPC Links per account'].quota_name == 'VPC links' def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -110,7 +112,7 @@ def test_find_usage(self): _find_usage_plans=DEFAULT, _find_usage_vpc_links=DEFAULT ) as mocks: - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -154,7 +156,7 @@ def se_get_stages(restApiId=None): mock_conn.get_paginator.side_effect = se_get_paginator mock_conn.get_stages.side_effect = se_get_stages - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.paginate_dict' % pbm, autospec=True) as mock_pd: with patch('%s.logger' % pbm) as mock_logger: @@ -351,7 +353,7 @@ def se_get_stages(restApiId=None): mock_conn.get_paginator.side_effect = se_get_paginator mock_conn.get_stages.side_effect = se_get_stages - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.paginate_dict' % pbm, autospec=True) as mock_pd: with patch('%s.logger' % pbm) as mock_logger: @@ -374,7 +376,7 @@ def test_find_usage_plans(self): mock_paginator.paginate.return_value = res mock_conn.get_paginator.return_value = mock_paginator - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % pbm) as mock_logger: cls._find_usage_plans() @@ -398,7 +400,7 @@ def test_find_usage_certs(self): mock_paginator.paginate.return_value = res mock_conn.get_paginator.return_value = mock_paginator - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % pbm) as mock_logger: cls._find_usage_certs() @@ -423,7 +425,7 @@ def test_find_usage_api_keys(self): mock_paginator.paginate.return_value = res mock_conn.get_paginator.return_value = mock_paginator - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % pbm) as mock_logger: cls._find_usage_api_keys() @@ -448,7 +450,7 @@ def test_find_usage_vpc_links(self): mock_paginator.paginate.return_value = res mock_conn.get_paginator.return_value = mock_paginator - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % pbm) as mock_logger: cls._find_usage_vpc_links() @@ -467,7 +469,7 @@ def test_find_usage_vpc_links(self): ] def test_required_iam_permissions(self): - cls = _ApigatewayService(21, 43) + cls = _ApigatewayService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "apigateway:GET", "apigateway:HEAD", diff --git a/awslimitchecker/tests/services/test_autoscaling.py b/awslimitchecker/tests/services/test_autoscaling.py index f852eaff..eca3f7dd 100644 --- a/awslimitchecker/tests/services/test_autoscaling.py +++ b/awslimitchecker/tests/services/test_autoscaling.py @@ -59,14 +59,14 @@ class Test_AutoscalingService(object): def test_init(self): """test __init__()""" - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) assert cls.service_name == 'AutoScaling' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -81,7 +81,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -109,7 +109,7 @@ def se_wrapper(func, *args, **kwargs): with patch('%s.connect' % self.pb) as mock_connect: with patch('%s.paginate_dict' % self.pbm) as mock_paginate: - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) cls.conn = mock_conn mock_paginate.side_effect = se_wrapper assert cls._have_usage is False @@ -139,7 +139,7 @@ def se_wrapper(func, *args, **kwargs): assert lcs[0].get_value() == 2 def test_required_iam_permissions(self): - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'autoscaling:DescribeAccountLimits', 'autoscaling:DescribeAutoScalingGroups', @@ -157,7 +157,7 @@ def test_update_limits_from_api(self): mock_conn.describe_account_limits.return_value = aslimits with patch('%s.connect' % self.pb) as mock_connect: - cls = _AutoscalingService(21, 43) + cls = _AutoscalingService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] diff --git a/awslimitchecker/tests/services/test_base.py b/awslimitchecker/tests/services/test_base.py index 739cef05..447df035 100644 --- a/awslimitchecker/tests/services/test_base.py +++ b/awslimitchecker/tests/services/test_base.py @@ -39,6 +39,7 @@ from awslimitchecker.services.base import _AwsService from awslimitchecker.limit import AwsLimit +from awslimitchecker.quotas import ServiceQuotasClient import pytest import sys @@ -48,9 +49,9 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock + from mock import patch, call, Mock, PropertyMock else: - from unittest.mock import patch, call, Mock + from unittest.mock import patch, call, Mock, PropertyMock class AwsServiceTester(_AwsService): @@ -77,7 +78,7 @@ class Test_AwsService(object): @pytest.mark.skipif(sys.version_info != (2, 7), reason='test for py27') def test_init_py27(self): with pytest.raises(TypeError) as excinfo: - _AwsService(1, 2) + _AwsService(1, 2, {}, None) assert excinfo.value.message == "Can't instantiate abstract class " \ "_AwsService with abstract methods " \ "connect" \ @@ -88,17 +89,19 @@ def test_init_py27(self): @pytest.mark.skipif(sys.version_info < (3, 0), reason='test for py3') def test_init_py3(self): with pytest.raises(NotImplementedError) as excinfo: - _AwsService(1, 2) + _AwsService(1, 2, {}, None) assert excinfo.value.args[0] == "abstract base class" def test_init_subclass(self): - cls = AwsServiceTester(1, 2) + m_quota = Mock() + cls = AwsServiceTester(1, 2, {}, m_quota) assert cls.warning_threshold == 1 assert cls.critical_threshold == 2 assert cls.limits == {'foo': 'bar'} assert cls.conn is None assert cls._have_usage is False - assert not cls._boto3_connection_kwargs + assert cls._boto3_connection_kwargs == {} + assert cls._quotas_client == m_quota def test_init_subclass_boto_xargs(self): boto_args = {'region_name': 'myregion', @@ -106,18 +109,19 @@ def test_init_subclass_boto_xargs(self): 'aws_secret_access_key': 'mysecretkey', 'aws_session_token': 'mytoken'} - cls = AwsServiceTester(1, 2, boto_args) + cls = AwsServiceTester(1, 2, boto_args, None) assert cls.warning_threshold == 1 assert cls.critical_threshold == 2 assert cls.limits == {'foo': 'bar'} assert cls.conn is None assert cls._have_usage is False assert cls._boto3_connection_kwargs == boto_args + assert cls._quotas_client is None def test_set_limit_override(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit cls.set_limit_override('foo', 10) assert mock_limit.mock_calls == [ @@ -127,7 +131,7 @@ def test_set_limit_override(self): def test_set_limit_override_keyerror(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit with pytest.raises(ValueError) as excinfo: cls.set_limit_override('bar', 10) @@ -144,7 +148,7 @@ def test_set_limit_override_keyerror(self): def test_set_ta_limit(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit cls._set_ta_limit('foo', 10) assert mock_limit.mock_calls == [ @@ -154,7 +158,7 @@ def test_set_ta_limit(self): def test_set_ta_limit_keyerror(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit with pytest.raises(ValueError) as excinfo: cls._set_ta_limit('bar', 10) @@ -171,7 +175,7 @@ def test_set_ta_limit_keyerror(self): def test_set_threshold_override(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit cls.set_threshold_override( 'foo', @@ -192,7 +196,7 @@ def test_set_threshold_override(self): def test_set_threshold_override_keyerror(self): mock_limit = Mock(spec_set=AwsLimit) type(mock_limit).default_limit = 5 - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.limits['foo'] = mock_limit with pytest.raises(ValueError) as excinfo: cls.set_threshold_override('bar', warn_percent=10) @@ -207,7 +211,7 @@ def test_set_threshold_override_keyerror(self): assert mock_limit.mock_calls == [] def test_check_thresholds(self): - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) cls.find_usage() mock_limit1 = Mock(spec_set=AwsLimit) mock_limit1.check_thresholds.return_value = False @@ -232,7 +236,7 @@ def test_check_thresholds(self): assert mock_find_usage.mock_calls == [] def test_check_thresholds_find_usage(self): - cls = AwsServiceTester(1, 2) + cls = AwsServiceTester(1, 2, {}, None) mock_limit1 = Mock(spec_set=AwsLimit) mock_limit1.check_thresholds.return_value = False cls.limits['foo'] = mock_limit1 @@ -255,6 +259,121 @@ def test_check_thresholds_find_usage(self): assert res == {'foo': mock_limit1, 'foo4': mock_limit4} assert mock_find_usage.mock_calls == [call()] + def test_update_service_quotas(self): + + def se_get_quota_value(_, quota_name, **kwargs): + if quota_name == 'qn1': + return 12.4 + return None + + mock_client = Mock(spec_set=ServiceQuotasClient) + mock_client.get_quota_value.side_effect = se_get_quota_value + mock_limit1 = Mock(spec_set=AwsLimit) + type(mock_limit1).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit1).quota_name = PropertyMock(return_value='qn1') + type(mock_limit1).quotas_unit = PropertyMock(return_value='None') + type(mock_limit1).quotas_unit_converter = PropertyMock( + return_value=None + ) + mock_limit2 = Mock(spec_set=AwsLimit) + type(mock_limit2).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit2).quota_name = PropertyMock(return_value='qn2') + type(mock_limit2).quotas_unit = PropertyMock(return_value='None') + type(mock_limit2).quotas_unit_converter = PropertyMock( + return_value=None + ) + mock_limit3 = Mock(spec_set=AwsLimit) + type(mock_limit3).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit3).quota_name = PropertyMock(return_value='qn3') + type(mock_limit3).quotas_unit = PropertyMock(return_value='Foo') + mock_conv = Mock() + type(mock_limit3).quotas_unit_converter = PropertyMock( + return_value=mock_conv + ) + cls = AwsServiceTester(1, 2, {}, mock_client) + cls.quotas_service_code = 'qsc' + cls.limits = { + 'limit1': mock_limit1, + 'limit2': mock_limit2, + 'limit3': mock_limit3 + } + cls._update_service_quotas() + assert mock_client.mock_calls == [ + call.get_quota_value('qsc', 'qn1', units='None', converter=None), + call.get_quota_value('qsc', 'qn2', units='None', converter=None), + call.get_quota_value( + 'qsc', 'qn3', units='Foo', converter=mock_conv + ) + ] + assert mock_limit1.mock_calls == [ + call._set_quotas_limit(12.4) + ] + assert mock_limit2.mock_calls == [] + + def test_update_service_quotas_no_code(self): + + def se_get_quota_value(_, quota_name, **kwargs): + if quota_name == 'qn1': + return 12.4 + return None + + mock_client = Mock(spec_set=ServiceQuotasClient) + mock_client.get_quota_value.side_effect = se_get_quota_value + mock_limit1 = Mock(spec_set=AwsLimit) + type(mock_limit1).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit1).quota_name = PropertyMock(return_value='qn1') + type(mock_limit1).quotas_unit = PropertyMock(return_value='None') + mock_limit2 = Mock(spec_set=AwsLimit) + type(mock_limit2).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit2).quota_name = PropertyMock(return_value='qn2') + type(mock_limit2).quotas_unit = PropertyMock(return_value='None') + cls = AwsServiceTester(1, 2, {}, mock_client) + cls.quotas_service_code = None + cls.limits = {'limit1': mock_limit1, 'limit2': mock_limit2} + cls._update_service_quotas() + assert mock_client.mock_calls == [] + assert mock_limit1.mock_calls == [] + assert mock_limit2.mock_calls == [] + + def test_update_service_quotas_no_client(self): + + def se_get_quota_value(_, quota_name, **kwargs): + if quota_name == 'qn1': + return 12.4 + return None + + mock_client = Mock(spec_set=ServiceQuotasClient) + mock_client.get_quota_value.side_effect = se_get_quota_value + mock_limit1 = Mock(spec_set=AwsLimit) + type(mock_limit1).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit1).quota_name = PropertyMock(return_value='qn1') + type(mock_limit1).quotas_unit = PropertyMock(return_value='None') + mock_limit2 = Mock(spec_set=AwsLimit) + type(mock_limit2).quotas_service_code = PropertyMock( + return_value='qsc' + ) + type(mock_limit2).quota_name = PropertyMock(return_value='qn2') + type(mock_limit2).quotas_unit = PropertyMock(return_value='None') + cls = AwsServiceTester(1, 2, {}, None) + cls.quotas_service_code = 'qsc' + cls.limits = {'limit1': mock_limit1, 'limit2': mock_limit2} + cls._update_service_quotas() + assert mock_client.mock_calls == [] + assert mock_limit1.mock_calls == [] + assert mock_limit2.mock_calls == [] + class Test_AwsServiceSubclasses(object): @@ -263,9 +382,11 @@ def test_subclass_init(self, cls): mock_limits = Mock() mock_get_limits = Mock() mock_get_limits.return_value = mock_limits + mock_quotas = Mock() with patch.object(cls, 'get_limits', mock_get_limits): - inst = cls(3, 7) + inst = cls(3, 7, {}, mock_quotas) assert inst.limits == mock_limits + assert inst._quotas_client == mock_quotas assert mock_get_limits.mock_calls == [call()] # ensure service name is changed assert inst.service_name != 'baseclass' @@ -282,5 +403,6 @@ def test_subclass_init(self, cls): aws_secret_access_key='mysecretkey', aws_session_token='mytoken') - sts_inst = cls(3, 7, boto_args) + sts_inst = cls(3, 7, boto_args, mock_quotas) assert sts_inst._boto3_connection_kwargs == boto_args + assert sts_inst._quotas_client == mock_quotas diff --git a/awslimitchecker/tests/services/test_cloudformation.py b/awslimitchecker/tests/services/test_cloudformation.py index 664db1a7..71aaecc6 100644 --- a/awslimitchecker/tests/services/test_cloudformation.py +++ b/awslimitchecker/tests/services/test_cloudformation.py @@ -59,7 +59,7 @@ class Test_CloudformationService(object): def test_init(self): """test __init__()""" - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) assert cls.service_name == 'CloudFormation' assert cls.api_name == 'cloudformation' assert cls.conn is None @@ -67,7 +67,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -82,7 +82,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -109,7 +109,7 @@ def test_find_usage(self): mock_conn = Mock() mock_conn.get_paginator.return_value = mock_paginator with patch('%s.connect' % pb) as mock_connect: - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -143,7 +143,7 @@ def test_update_limits_from_api(self): } with patch('%s.connect' % pb) as mock_connect: - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -151,7 +151,7 @@ def test_update_limits_from_api(self): assert cls.limits['Stacks'].api_limit == 400 def test_required_iam_permissions(self): - cls = _CloudformationService(21, 43) + cls = _CloudformationService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'cloudformation:DescribeAccountLimits', 'cloudformation:DescribeStacks' diff --git a/awslimitchecker/tests/services/test_cloudtrail.py b/awslimitchecker/tests/services/test_cloudtrail.py index 20d5ea27..b97b05d3 100644 --- a/awslimitchecker/tests/services/test_cloudtrail.py +++ b/awslimitchecker/tests/services/test_cloudtrail.py @@ -62,7 +62,7 @@ class Test_CloudTrailService(object): def test_init(self): with patch('%s.get_limits' % PATCH_BASE): - cls = _CloudTrailService(21, 43) + cls = _CloudTrailService(21, 43, {}, None) assert cls.service_name == 'CloudTrail' assert cls.api_name == 'cloudtrail' assert cls.conn is None @@ -71,7 +71,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _CloudTrailService(21, 43) + cls = _CloudTrailService(21, 43, {}, None) limit_dict = cls.get_limits() for limit in limit_dict: @@ -99,7 +99,7 @@ def test_get_limits(self): def test_get_limits_again(self): """Test that existing limits dict is returned on subsequent calls""" mock_limits = Mock(spec_set=AwsLimit) - cls = _CloudTrailService(21, 43) + cls = _CloudTrailService(21, 43, {}, None) cls.limits = mock_limits response = cls.get_limits() assert response == mock_limits @@ -113,17 +113,19 @@ def test_find_usage(self): result_fixtures.CloudTrail.mock_describe_trails def se_selectors(*args, **kwargs): - if kwargs['TrailName'] == 'trail2': + if kwargs['TrailName'] == 'trailarn2': return result_fixtures.CloudTrail.mock_get_event_selectors + if kwargs['TrailName'] == 'trailarn4': + raise RuntimeError('foo') return { - 'TrailArn': 'arn:%s' % kwargs['TrailName'], + 'TrailArn': kwargs['TrailName'], 'EventSelectors': [] } mock_trails.get_event_selectors.side_effect = se_selectors with patch('%s.connect' % PATCH_BASE,) as mock_connect: - cls = _CloudTrailService(21, 43) + cls = _CloudTrailService(21, 43, {}, None) cls.conn = mock_trails assert cls._have_usage is False cls.find_usage() @@ -133,7 +135,7 @@ def se_selectors(*args, **kwargs): usage = cls.limits['Trails Per Region'].get_current_usage() assert len(usage) == 1 - assert usage[0].get_value() == 3 + assert usage[0].get_value() == 4 usage = cls.limits['Event Selectors Per Trail'].get_current_usage() assert len(usage) == 2 @@ -146,7 +148,7 @@ def se_selectors(*args, **kwargs): assert usage[1].get_value() == 3 def test_required_iam_permissions(self): - cls = _CloudTrailService(21, 43) + cls = _CloudTrailService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "cloudtrail:DescribeTrails", "cloudtrail:GetEventSelectors", diff --git a/awslimitchecker/tests/services/test_directoryservice.py b/awslimitchecker/tests/services/test_directoryservice.py index 1f926a0a..138b8aa1 100644 --- a/awslimitchecker/tests/services/test_directoryservice.py +++ b/awslimitchecker/tests/services/test_directoryservice.py @@ -60,7 +60,7 @@ class Test_DirectoryserviceService(object): def test_init(self): """test __init__()""" - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) assert cls.service_name == 'Directory Service' assert cls.api_name == 'ds' assert cls.conn is None @@ -68,7 +68,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -84,7 +84,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -105,7 +105,7 @@ def test_find_usage(self): } } with patch('%s.connect' % pb) as mock_connect: - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -135,7 +135,7 @@ def test_update_limits_from_api(self): } } with patch('%s.connect' % pb) as mock_connect: - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls._update_limits_from_api() @@ -146,7 +146,7 @@ def test_update_limits_from_api(self): assert cls.limits['ConnectedDirectories'].api_limit == 12 def test_required_iam_permissions(self): - cls = _DirectoryserviceService(21, 43) + cls = _DirectoryserviceService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'ds:GetDirectoryLimits' ] diff --git a/awslimitchecker/tests/services/test_dynamodb.py b/awslimitchecker/tests/services/test_dynamodb.py index b8572bf6..9df760ba 100644 --- a/awslimitchecker/tests/services/test_dynamodb.py +++ b/awslimitchecker/tests/services/test_dynamodb.py @@ -62,7 +62,7 @@ class Test_DynamodbService(object): def test_init(self): """test __init__()""" with patch('%s.get_limits' % pb): - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) assert cls.service_name == 'DynamoDB' assert cls.api_name == 'dynamodb' assert cls.conn is None @@ -81,7 +81,7 @@ def se_conn(cls): with patch('%s.connect' % pb, autospec=True) as mock_connect: mock_connect.side_effect = se_conn - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) limits = cls.limits for x in limits: @@ -123,7 +123,7 @@ def se_conn(cls): with patch('%s.connect' % pb, autospec=True) as mock_connect: mock_connect.side_effect = se_conn - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) limits = cls.limits for x in limits: @@ -157,7 +157,7 @@ def se_conn(cls): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock(spec_set=AwsLimit) - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -175,7 +175,7 @@ def se_conn(cls): with patch('%s.connect' % pb, autospec=True) as mock_connect: mock_connect.side_effect = se_conn - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert cls.limits['Account Max Read Capacity Units'].api_limit == 111 @@ -202,7 +202,7 @@ def se_conn(cls): '%s.connect_resource' % pb, autospec=True ) as mock_conn_res: mock_connect.side_effect = se_conn - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -231,7 +231,7 @@ def se_conn(cls): with patch('%s.connect' % pb, autospec=True) as mock_connect: mock_connect.side_effect = se_conn - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) cls.conn = mock_conn cls.resource_conn = mock_res_conn cls._find_usage_dynamodb() @@ -280,7 +280,7 @@ def se_conn(cls): assert u[2].get_value() == 600 def test_required_iam_permissions(self): - cls = _DynamodbService(21, 43) + cls = _DynamodbService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "dynamodb:DescribeLimits", "dynamodb:DescribeTable", diff --git a/awslimitchecker/tests/services/test_ebs.py b/awslimitchecker/tests/services/test_ebs.py index 76f6d2fe..ee0357da 100644 --- a/awslimitchecker/tests/services/test_ebs.py +++ b/awslimitchecker/tests/services/test_ebs.py @@ -38,7 +38,7 @@ """ import sys -from awslimitchecker.services.ebs import _EbsService +from awslimitchecker.services.ebs import _EbsService, convert_TiB_to_GiB from awslimitchecker.limit import AwsLimit from awslimitchecker.tests.services import result_fixtures @@ -53,6 +53,15 @@ from unittest.mock import patch, call, Mock, DEFAULT +class TestConvertTiBToGiB(object): + + def test_happy_path(self): + assert convert_TiB_to_GiB(300.0, 'None', 'GiB') == 307200.0 + + def test_unknown_unit(self): + assert convert_TiB_to_GiB(300.0, 'Foo', 'GiB') is None + + class Test_EbsService(object): pb = 'awslimitchecker.services.ebs._EbsService' # patch base path @@ -60,7 +69,7 @@ class Test_EbsService(object): def test_init(self): """test __init__()""" - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) assert cls.service_name == 'EBS' assert cls.conn is None assert cls.warning_threshold == 21 @@ -68,7 +77,7 @@ def test_init(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) cls.limits = {'foo': 'bar'} with patch('%s._get_limits_ebs' % self.pb) as mock_ebs: res = cls.get_limits() @@ -77,7 +86,7 @@ def test_get_limits_again(self): def test_get_limits(self): """test some things all limits should conform to""" - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) limits = cls.get_limits() for x in limits: assert isinstance(limits[x], AwsLimit) @@ -91,17 +100,17 @@ def test_get_limits(self): piops_tb = limits['Provisioned IOPS (SSD) storage (GiB)'] assert piops_tb.limit_type == 'AWS::EC2::Volume' assert piops_tb.limit_subtype == 'io1' - assert piops_tb.default_limit == 102400 + assert piops_tb.default_limit == 307200 gp_tb = limits['General Purpose (SSD) volume storage (GiB)'] assert gp_tb.limit_type == 'AWS::EC2::Volume' assert gp_tb.limit_subtype == 'gp2' - assert gp_tb.default_limit == 102400 + assert gp_tb.default_limit == 307200 assert gp_tb.ta_limit_name == 'General Purpose SSD (gp2) ' \ 'volume storage (GiB)' mag_tb = limits['Magnetic volume storage (GiB)'] assert mag_tb.limit_type == 'AWS::EC2::Volume' assert mag_tb.limit_subtype == 'standard' - assert mag_tb.default_limit == 20480 + assert mag_tb.default_limit == 307200 assert mag_tb.ta_limit_name == 'Magnetic (standard) volume ' \ 'storage (GiB)' st_tb = limits['Throughput Optimized (HDD) volume storage (GiB)'] @@ -125,7 +134,7 @@ def test_find_usage(self): _find_usage_snapshots=DEFAULT, autospec=True, ) as mocks: - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) assert cls._have_usage is False cls.find_usage() assert cls._have_usage is True @@ -137,7 +146,7 @@ def test_find_usage_ebs(self): response = result_fixtures.EBS.test_find_usage_ebs mock_conn = Mock() - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) cls.conn = mock_conn with patch('awslimitchecker.services.ebs.logger') as mock_logger: with patch('%s.paginate_dict' % self.pbm) as mock_paginate: @@ -191,7 +200,7 @@ def test_find_usage_snapshots(self): mock_conn = Mock() - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) cls.conn = mock_conn with patch('awslimitchecker.services.ebs.logger') as mock_logger: with patch('%s.paginate_dict' % self.pbm) as mock_paginate: @@ -215,7 +224,7 @@ def test_find_usage_snapshots(self): ] def test_required_iam_permissions(self): - cls = _EbsService(21, 43) + cls = _EbsService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "ec2:DescribeVolumes", "ec2:DescribeSnapshots" diff --git a/awslimitchecker/tests/services/test_ec2.py b/awslimitchecker/tests/services/test_ec2.py index 8e7d52bf..533d8756 100644 --- a/awslimitchecker/tests/services/test_ec2.py +++ b/awslimitchecker/tests/services/test_ec2.py @@ -37,6 +37,7 @@ ################################################################################ """ +import os import sys from copy import deepcopy import pytest @@ -52,33 +53,36 @@ sys.version_info[0] < 3 or sys.version_info[0] == 3 and sys.version_info[1] < 4 ): - from mock import patch, call, Mock, DEFAULT + from mock import patch, call, Mock, DEFAULT, PropertyMock else: - from unittest.mock import patch, call, Mock, DEFAULT + from unittest.mock import patch, call, Mock, DEFAULT, PropertyMock fixtures = result_fixtures.EC2() +pb = 'awslimitchecker.services.ec2._Ec2Service' # patch base path -class Test_Ec2Service(object): +class TestInit(object): - pb = 'awslimitchecker.services.ec2._Ec2Service' # patch base path - pbm = 'awslimitchecker.services.ec2' # module patch base path - - def test_init(self): + def test_simple(self): """test __init__()""" - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) assert cls.service_name == 'EC2' assert cls.conn is None assert cls.resource_conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 + assert cls.quotas_service_code == 'ec2' + - def test_instance_types(self): - cls = _Ec2Service(21, 43) +class TestInstanceTypes(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) types = cls._instance_types() # NOTE hi1.4xlarge is no longer in the instance type listings, # but some accounts might still have a limit for it - assert len(types) == 175 + assert len(set(types)) == len(types) + assert len(types) == 268 assert 't2.micro' in types assert 'r3.8xlarge' in types assert 'c3.large' in types @@ -93,52 +97,146 @@ def test_instance_types(self): assert 'm4.16xlarge' in types assert 'x1.32xlarge' in types assert 'z1d.12xlarge' in types + assert 'u-24tb1.metal' in types + assert 'm5n.metal' in types + - def test_get_limits(self): - cls = _Ec2Service(21, 43) +class TestGetLimits(object): + + def test_nonvcpu(self): + cls = _Ec2Service(21, 43, {}, None) cls.limits = {} - with patch('%s._get_limits_instances' % self.pb) as mock_instances: - with patch('%s._get_limits_networking' % self.pb) as mock_vpc: - with patch('%s._get_limits_spot' % self.pb) as mock_spot: - mock_instances.return_value = {'ec2lname': 'ec2lval'} - mock_vpc.return_value = {'vpck': 'vpcv'} - mock_spot.return_value = {'spotk': 'spotv'} - res = cls.get_limits() + with patch.multiple( + pb, + _get_limits_instances_nonvcpu=DEFAULT, + _get_limits_instances_vcpu=DEFAULT, + _get_limits_networking=DEFAULT, + _get_limits_spot=DEFAULT, + autospec=True + ) as mocks: + mocks['_get_limits_instances_nonvcpu'].return_value = { + 'ec2lname': 'ec2lval' + } + mocks['_get_limits_instances_vcpu'].return_value = { + 'fooname': 'fooval' + } + mocks['_get_limits_networking'].return_value = {'vpck': 'vpcv'} + mocks['_get_limits_spot'].return_value = {'spotk': 'spotv'} + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + res = cls.get_limits() assert res == { 'ec2lname': 'ec2lval', 'spotk': 'spotv', 'vpck': 'vpcv', } - assert mock_instances.mock_calls == [call()] - assert mock_vpc.mock_calls == [call()] - assert mock_spot.mock_calls == [call()] + assert mocks['_get_limits_instances_nonvcpu'].mock_calls == [ + call(cls) + ] + assert mocks['_get_limits_instances_vcpu'].mock_calls == [] + assert mocks['_get_limits_networking'].mock_calls == [call(cls)] + assert mocks['_get_limits_spot'].mock_calls == [call(cls)] + + def test_vcpu(self): + cls = _Ec2Service(21, 43, {}, None) + cls.limits = {} + with patch.multiple( + pb, + _get_limits_instances_nonvcpu=DEFAULT, + _get_limits_instances_vcpu=DEFAULT, + _get_limits_networking=DEFAULT, + _get_limits_spot=DEFAULT, + autospec=True + ) as mocks: + mocks['_get_limits_instances_nonvcpu'].return_value = { + 'ec2lname': 'ec2lval' + } + mocks['_get_limits_instances_vcpu'].return_value = { + 'fooname': 'fooval' + } + mocks['_get_limits_networking'].return_value = {'vpck': 'vpcv'} + mocks['_get_limits_spot'].return_value = {'spotk': 'spotv'} + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = True + res = cls.get_limits() + assert res == { + 'fooname': 'fooval', + 'spotk': 'spotv', + 'vpck': 'vpcv', + } + assert mocks['_get_limits_instances_nonvcpu'].mock_calls == [] + assert mocks['_get_limits_instances_vcpu'].mock_calls == [call(cls)] + assert mocks['_get_limits_networking'].mock_calls == [call(cls)] + assert mocks['_get_limits_spot'].mock_calls == [call(cls)] - def test_get_limits_again(self): + def test_get_again(self): """test that existing limits dict is returned on subsequent calls""" - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.limits = {'foo': 'bar'} - with patch('%s._get_limits_instances' % self.pb) as mock_instances: - with patch('%s._get_limits_networking' % self.pb) as mock_vpc: - with patch('%s._get_limits_spot' % self.pb) as mock_spot: - res = cls.get_limits() + with patch.multiple( + pb, + _get_limits_instances_nonvcpu=DEFAULT, + _get_limits_instances_vcpu=DEFAULT, + _get_limits_networking=DEFAULT, + _get_limits_spot=DEFAULT, + autospec=True + ) as mocks: + mocks['_get_limits_instances_nonvcpu'].return_value = { + 'ec2lname': 'ec2lval' + } + mocks['_get_limits_instances_vcpu'].return_value = { + 'fooname': 'fooval' + } + mocks['_get_limits_networking'].return_value = {'vpck': 'vpcv'} + mocks['_get_limits_spot'].return_value = {'spotk': 'spotv'} + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + res = cls.get_limits() assert res == {'foo': 'bar'} - assert mock_instances.mock_calls == [] - assert mock_vpc.mock_calls == [] - assert mock_spot.mock_calls == [] + assert mocks['_get_limits_instances_nonvcpu'].mock_calls == [] + assert mocks['_get_limits_instances_vcpu'].mock_calls == [] + assert mocks['_get_limits_networking'].mock_calls == [] + assert mocks['_get_limits_spot'].mock_calls == [] - def test_get_limits_all(self): + def test_all_nonvcpu(self): """test some things all limits should conform to""" - cls = _Ec2Service(21, 43) - limits = cls.get_limits() + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + cls = _Ec2Service(21, 43, {}, None) + limits = cls.get_limits() for x in limits: assert isinstance(limits[x], AwsLimit) assert x == limits[x].name assert limits[x].service == cls - def test_get_limits_instances(self): - cls = _Ec2Service(21, 43) - limits = cls._get_limits_instances() - assert len(limits) == 176 + def test_all_vcpu(self): + """test some things all limits should conform to""" + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = True + cls = _Ec2Service(21, 43, {}, None) + limits = cls.get_limits() + for x in limits: + assert isinstance(limits[x], AwsLimit) + assert x == limits[x].name + assert limits[x].service == cls + + +class TestGetLimitsInstancesNonvcpu(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) + limits = cls._get_limits_instances_nonvcpu() + assert len(limits) == 269 # check a random subset of limits t2_micro = limits['Running On-Demand t2.micro instances'] assert t2_micro.default_limit == 20 @@ -176,11 +274,75 @@ def test_get_limits_instances(self): assert lname == 'Running On-Demand %s instances' % itype assert lim.ta_limit_name == 'On-Demand instances - %s' % itype - def test_find_usage(self): + +class TestGetLimitsInstancesVcpu(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) + limits = cls._get_limits_instances_vcpu() + assert len(limits) == 5 + for k in ['f', 'g', 'p', 'x']: + lim = limits['Running On-Demand All %s instances' % k.upper()] + assert lim.default_limit == 128 + assert lim.limit_type == 'On-Demand instances' + assert lim.limit_subtype == k.upper() + k = 'Running On-Demand All Standard ' \ + '(A, C, D, H, I, M, R, T, Z) instances' + lim = limits[k] + assert lim.default_limit == 1152 + assert lim.limit_type == 'On-Demand instances' + assert lim.limit_subtype == 'Standard' + + +class TestFindUsage(object): + + def test_nonvcpu(self): + with patch.multiple( + pb, + connect=DEFAULT, + _find_usage_instances_nonvcpu=DEFAULT, + _find_usage_instances_vcpu=DEFAULT, + _find_usage_networking_sgs=DEFAULT, + _find_usage_networking_eips=DEFAULT, + _find_usage_networking_eni_sg=DEFAULT, + _find_usage_spot_instances=DEFAULT, + _find_usage_spot_fleets=DEFAULT, + autospec=True, + ) as mocks: + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + cls = _Ec2Service(21, 43, {}, None) + assert cls._have_usage is False + cls.find_usage() + assert cls._have_usage is True + assert mocks['_find_usage_instances_nonvcpu'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_instances_vcpu'].mock_calls == [] + assert mocks['_find_usage_networking_sgs'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_networking_eips'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_networking_eni_sg'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_spot_instances'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_spot_fleets'].mock_calls == [ + call(cls) + ] + + def test_vcpu(self): with patch.multiple( - self.pb, + pb, connect=DEFAULT, - _find_usage_instances=DEFAULT, + _find_usage_instances_nonvcpu=DEFAULT, + _find_usage_instances_vcpu=DEFAULT, _find_usage_networking_sgs=DEFAULT, _find_usage_networking_eips=DEFAULT, _find_usage_networking_eni_sg=DEFAULT, @@ -188,15 +350,38 @@ def test_find_usage(self): _find_usage_spot_fleets=DEFAULT, autospec=True, ) as mocks: - cls = _Ec2Service(21, 43) - assert cls._have_usage is False - cls.find_usage() + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = True + cls = _Ec2Service(21, 43, {}, None) + assert cls._have_usage is False + cls.find_usage() assert cls._have_usage is True - assert len(mocks) == 7 - for m in mocks: - assert mocks[m].mock_calls == [call(cls)] + assert mocks['_find_usage_instances_nonvcpu'].mock_calls == [] + assert mocks['_find_usage_instances_vcpu'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_networking_sgs'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_networking_eips'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_networking_eni_sg'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_spot_instances'].mock_calls == [ + call(cls) + ] + assert mocks['_find_usage_spot_fleets'].mock_calls == [ + call(cls) + ] + - def test_instance_usage(self): +class TestInstanceUsage(object): + + def test_simple(self): mock_t2_micro = Mock(spec_set=AwsLimit) mock_r3_2xlarge = Mock(spec_set=AwsLimit) mock_c4_4xlarge = Mock(spec_set=AwsLimit) @@ -208,7 +393,7 @@ def test_instance_usage(self): 'Running On-Demand m4.8xlarge instances': mock_m4_8xlarge, } - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) mock_conn = Mock() retval = fixtures.test_instance_usage @@ -238,10 +423,93 @@ def test_instance_usage(self): call.instances.all() ] - def test_get_reserved_instance_count(self): + def test_key_error(self): + mock_conn = Mock() + data = fixtures.test_instance_usage_key_error + mock_conn.instances.all.return_value = data + cls = _Ec2Service(21, 43, {}, None) + cls.resource_conn = mock_conn + cls.limits = {'Running On-Demand t2.micro instances': Mock()} + + with patch( + '%s._instance_types' % pb, + autospec=True) as mock_itypes: + with patch('awslimitchecker.services.ec2.logger') as mock_logger: + mock_itypes.return_value = ['t2.micro'] + cls._instance_usage() + assert mock_logger.mock_calls == [ + call.debug('Getting usage for on-demand instances'), + call.error("ERROR - unknown instance type '%s'; not counting", + 'foobar'), + ] + assert mock_conn.mock_calls == [ + call.instances.all() + ] + + +class TestInstanceUsageVcpu(object): + + def test_no_RIs(self): + cls = _Ec2Service(21, 43, {}, None) + mock_conn = Mock() + retval = fixtures.test_instance_usage_vcpu + mock_conn.instances.all.return_value = retval + cls.resource_conn = mock_conn + + res = cls._instance_usage_vcpu({}) + assert res == { + 'c': 16, + 'f': 72, + 'g': 48, + 'm': 32, + 'r': 16, + 't': 2, + 'p': 128, + 'x': 256, + } + assert mock_conn.mock_calls == [ + call.instances.all() + ] + + def test_with_RIs(self): + cls = _Ec2Service(21, 43, {}, None) + mock_conn = Mock() + retval = fixtures.test_instance_usage_vcpu + mock_conn.instances.all.return_value = retval + cls.resource_conn = mock_conn + + res = cls._instance_usage_vcpu({ + 'az1a': { + 'f1.2xlarge': 10, + 'c4.4xlarge': 8, + 'c4.2xlarge': 16, + }, + 'az1c': { + 'x1e.32xlarge': 1, + 'p2.8xlarge': 1, + 'p2.16xlarge': 1 + } + }) + assert res == { + 'f': 64, + 'g': 48, + 'm': 32, + 'r': 16, + 't': 2, + 'p': 32, + 'x': 128, + } + assert mock_conn.mock_calls == [ + call.instances.all() + ] + + +class TestGetReservedInstanceCount(object): + + def test_simple(self): response = fixtures.test_get_reserved_instance_count - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) mock_client_conn = Mock() cls.conn = mock_client_conn mock_client_conn.describe_reserved_instances.return_value = response @@ -266,7 +534,10 @@ def test_get_reserved_instance_count(self): call.describe_reserved_instances() ] - def test_find_usage_instances(self): + +class TestFindUsageInstancesNonvcpu(object): + + def test_simple(self): iusage = { 'us-east-1': { 't2.micro': 2, @@ -313,17 +584,17 @@ def test_find_usage_instances(self): 'Running On-Demand EC2 instances': mock_all_ec2, } - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) mock_conn = Mock() cls.resource_conn = mock_conn cls.limits = limits - with patch('%s._instance_usage' % self.pb, + with patch('%s._instance_usage' % pb, autospec=True) as mock_inst_usage: - with patch('%s._get_reserved_instance_count' % self.pb, + with patch('%s._get_reserved_instance_count' % pb, autospec=True) as mock_res_inst_count: mock_inst_usage.return_value = iusage mock_res_inst_count.return_value = ri_count - cls._find_usage_instances() + cls._find_usage_instances_nonvcpu() assert mock_t2_micro.mock_calls == [call._add_current_usage( 36, aws_type='AWS::EC2::Instance' @@ -348,31 +619,124 @@ def test_find_usage_instances(self): assert mock_res_inst_count.mock_calls == [call(cls)] assert mock_conn.mock_calls == [] - def test_instance_usage_key_error(self): - mock_conn = Mock() - data = fixtures.test_instance_usage_key_error - mock_conn.instances.all.return_value = data - cls = _Ec2Service(21, 43) - cls.resource_conn = mock_conn - cls.limits = {'Running On-Demand t2.micro instances': Mock()} + +class TestFindUsageInstancesVcpu(object): + + def test_happy_path(self): + usage = { + 'c': 16, + 'f': 72, + 'g': 48, + 'm': 32, + 'r': 16, + 't': 2, + 'p': 128, + 'x': 256, + 'a': 512, + 'k': 3 + } + + mock_f = Mock(spec_set=AwsLimit) + mock_g = Mock(spec_set=AwsLimit) + mock_p = Mock(spec_set=AwsLimit) + mock_x = Mock(spec_set=AwsLimit) + mock_std = Mock(spec_set=AwsLimit) + limits = { + 'Running On-Demand All F instances': mock_f, + 'Running On-Demand All G instances': mock_g, + 'Running On-Demand All P instances': mock_p, + 'Running On-Demand All X instances': mock_x, + 'Running On-Demand All Standard ' + '(A, C, D, H, I, M, R, T, Z) instances': mock_std + } + + cls = _Ec2Service(21, 43, {}, None) + cls.limits = limits with patch( - '%s._instance_types' % self.pb, - autospec=True) as mock_itypes: - with patch('awslimitchecker.services.ec2.logger') as mock_logger: - mock_itypes.return_value = ['t2.micro'] - cls._instance_usage() - assert mock_logger.mock_calls == [ - call.debug('Getting usage for on-demand instances'), - call.error("ERROR - unknown instance type '%s'; not counting", - 'foobar'), + '%s._get_reserved_instance_count' % pb, autospec=True + ) as m_gric: + with patch('%s._instance_usage_vcpu' % pb, autospec=True) as m_iuv: + m_gric.return_value = {'res': 'inst'} + m_iuv.return_value = usage + cls._find_usage_instances_vcpu() + assert m_gric.mock_calls == [call(cls)] + assert m_iuv.mock_calls == [call(cls, {'res': 'inst'})] + assert mock_f.mock_calls == [ + call._add_current_usage(72, aws_type='AWS::EC2::Instance') ] - assert mock_conn.mock_calls == [ - call.instances.all() + assert mock_g.mock_calls == [ + call._add_current_usage(48, aws_type='AWS::EC2::Instance') + ] + assert mock_p.mock_calls == [ + call._add_current_usage(128, aws_type='AWS::EC2::Instance') + ] + assert mock_x.mock_calls == [ + call._add_current_usage(256, aws_type='AWS::EC2::Instance') + ] + assert mock_std.mock_calls == [ + call._add_current_usage(581, aws_type='AWS::EC2::Instance') + ] + + def test_default_zero(self): + usage = { + 'c': 16, + 'm': 32, + 'r': 16, + 't': 2, + 'p': 128, + 'x': 256, + 'a': 512, + 'k': 3 + } + + mock_f = Mock(spec_set=AwsLimit) + mock_g = Mock(spec_set=AwsLimit) + mock_p = Mock(spec_set=AwsLimit) + mock_x = Mock(spec_set=AwsLimit) + mock_std = Mock(spec_set=AwsLimit) + limits = { + 'Running On-Demand All F instances': mock_f, + 'Running On-Demand All G instances': mock_g, + 'Running On-Demand All P instances': mock_p, + 'Running On-Demand All X instances': mock_x, + 'Running On-Demand All Standard ' + '(A, C, D, H, I, M, R, T, Z) instances': mock_std + } + + cls = _Ec2Service(21, 43, {}, None) + cls.limits = limits + + with patch( + '%s._get_reserved_instance_count' % pb, autospec=True + ) as m_gric: + with patch('%s._instance_usage_vcpu' % pb, autospec=True) as m_iuv: + m_gric.return_value = {'res': 'inst'} + m_iuv.return_value = usage + cls._find_usage_instances_vcpu() + assert m_gric.mock_calls == [call(cls)] + assert m_iuv.mock_calls == [call(cls, {'res': 'inst'})] + assert mock_f.mock_calls == [ + call._add_current_usage(0, aws_type='AWS::EC2::Instance') + ] + assert mock_g.mock_calls == [ + call._add_current_usage(0, aws_type='AWS::EC2::Instance') + ] + assert mock_p.mock_calls == [ + call._add_current_usage(128, aws_type='AWS::EC2::Instance') + ] + assert mock_x.mock_calls == [ + call._add_current_usage(256, aws_type='AWS::EC2::Instance') + ] + assert mock_std.mock_calls == [ + call._add_current_usage(581, aws_type='AWS::EC2::Instance') ] - def test_required_iam_permissions(self): - cls = _Ec2Service(21, 43) + +class TestRequiredIamPermissions(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) assert len(cls.required_iam_permissions()) == 19 assert cls.required_iam_permissions() == [ "ec2:DescribeAccountAttributes", @@ -396,13 +760,16 @@ def test_required_iam_permissions(self): "ec2:DescribeVpcs", ] - def test_find_usage_networking_sgs(self): + +class TestFindUsageNetworkingSgs(object): + + def test_simple(self): mocks = fixtures.test_find_usage_networking_sgs mock_conn = Mock() mock_conn.security_groups.all.return_value = mocks - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: @@ -430,22 +797,29 @@ def test_find_usage_networking_sgs(self): assert sorted_usage[0].resource_id == 'sg-1' assert sorted_usage[0].get_value() == 0 assert sorted_usage[1].limit == limit - assert sorted_usage[1].resource_id == 'sg-4' - assert sorted_usage[1].get_value() == 3 + assert sorted_usage[1].resource_id == 'sg-2' + # ingress: IPv4 = 15; IPv6 = 13 + # egress: IPv4 = 5; IPv6 = 7 + assert sorted_usage[1].get_value() == 15 assert sorted_usage[2].limit == limit assert sorted_usage[2].resource_id == 'sg-3' - assert sorted_usage[2].get_value() == 9 + # ingress: IPv4 = 4; IPv6 = 5 + # egress: IPv4 = 22; IPv6 = 29 + assert sorted_usage[2].get_value() == 29 assert mock_conn.mock_calls == [ call.security_groups.all() ] - def test_find_usage_networking_eips(self): + +class TestFindUsageNetworkingEips(object): + + def test_simple(self): mocks = fixtures.test_find_usage_networking_eips mock_conn = Mock() mock_conn.classic_addresses.all.return_value = mocks['Classic'] mock_conn.vpc_addresses.all.return_value = mocks['Vpc'] - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: @@ -474,12 +848,15 @@ def test_find_usage_networking_eips(self): call.classic_addresses.all() ] - def test_find_usage_networking_eni_sg(self): + +class TestFindUsageNetworkingEniSg(object): + + def test_simple(self): mocks = fixtures.test_find_usage_networking_eni_sg mock_conn = Mock() mock_conn.network_interfaces.all.return_value = mocks - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: cls._find_usage_networking_eni_sg() @@ -502,8 +879,11 @@ def test_find_usage_networking_eni_sg(self): call.network_interfaces.all() ] - def test_get_limits_networking(self): - cls = _Ec2Service(21, 43) + +class TestGetLimitsNetworking(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) limits = cls._get_limits_networking() expected = [ 'Security groups per VPC', @@ -514,10 +894,31 @@ def test_get_limits_networking(self): ] assert sorted(limits.keys()) == sorted(expected) assert limits[ - 'VPC Elastic IP addresses (EIPs)'].ta_service_name == 'VPC' + 'VPC Elastic IP addresses (EIPs)'].ta_service_name == 'VPC' + assert limits[ + 'VPC Elastic IP addresses (EIPs)' + ].quotas_service_code == 'ec2' + assert limits[ + 'VPC Elastic IP addresses (EIPs)' + ].quota_name == 'Number of EIPs - VPC EIPs' + assert limits[ + 'VPC Elastic IP addresses (EIPs)' + ].quotas_unit == 'None' + assert limits[ + 'Elastic IP addresses (EIPs)' + ].quotas_service_code == 'ec2' + assert limits[ + 'Elastic IP addresses (EIPs)' + ].quota_name == 'Elastic IP addresses for EC2-Classic' + assert limits[ + 'Elastic IP addresses (EIPs)' + ].quotas_unit == 'None' - def test_get_limits_spot(self): - cls = _Ec2Service(21, 43) + +class TestGetLimitsSpot(object): + + def test_simple(self): + cls = _Ec2Service(21, 43, {}, None) limits = cls._get_limits_spot() expected = [ 'Max spot instance requests per region', @@ -528,12 +929,15 @@ def test_get_limits_spot(self): ] assert sorted(limits.keys()) == sorted(expected) - def test_find_usage_spot_instances(self): + +class TestFindUsageSpotInstances(object): + + def test_happy_path(self): data = fixtures.test_find_usage_spot_instances mock_conn = Mock() mock_client_conn = Mock() mock_client_conn.describe_spot_instance_requests.return_value = data - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn cls.conn = mock_client_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: @@ -558,47 +962,50 @@ def test_find_usage_spot_instances(self): 'reqID4', 'failed') ] - def test_find_usage_spot_instances_unsupported(self): + def test_unsupported(self): mock_client_conn = Mock() err = botocore.exceptions.ClientError( {'Error': {'Code': 'UnsupportedOperation'}}, 'operation', ) mock_client_conn.describe_spot_instance_requests.side_effect = err - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn cls._find_usage_spot_instances() lim = cls.limits['Max spot instance requests per region'] usage = lim.get_current_usage() assert len(usage) == 0 - def test_find_usage_spot_instances_unknown_code(self): + def test_unknown_code(self): mock_client_conn = Mock() err = botocore.exceptions.ClientError( {'Error': {'Code': 'SomeCode'}}, 'operation', ) mock_client_conn.describe_spot_instance_requests.side_effect = err - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn with pytest.raises(botocore.exceptions.ClientError): cls._find_usage_spot_instances() - def test_find_usage_spot_instances_unknown_error(self): + def test_unknown_error(self): mock_client_conn = Mock() err = RuntimeError mock_client_conn.describe_spot_instance_requests.side_effect = err - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn with pytest.raises(RuntimeError): cls._find_usage_spot_instances() - def test_find_usage_spot_fleets(self): + +class TestFindUsageSpotFleets(object): + + def test_simple(self): data = fixtures.test_find_usage_spot_fleets mock_conn = Mock() mock_client_conn = Mock() mock_client_conn.describe_spot_fleet_requests.return_value = data - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn cls.conn = mock_client_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: @@ -642,13 +1049,13 @@ def test_find_usage_spot_fleets(self): 'req3', 'modifying') ] - def test_find_usage_spot_fleets_paginated(self): + def test_paginated(self): data = deepcopy(fixtures.test_find_usage_spot_fleets) data['NextToken'] = 'string' mock_conn = Mock() mock_client_conn = Mock() mock_client_conn.describe_spot_fleet_requests.return_value = data - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.resource_conn = mock_conn cls.conn = mock_client_conn with patch('awslimitchecker.services.ec2.logger') as mock_logger: @@ -695,51 +1102,58 @@ def test_find_usage_spot_fleets_paginated(self): 'req3', 'modifying') ] - def test_find_usage_spot_fleets_unsupported(self): + def test_unsupported(self): mock_client_conn = Mock() err = botocore.exceptions.ClientError( {'Error': {'Code': 'UnsupportedOperation'}}, 'operation', ) mock_client_conn.describe_spot_fleet_requests.side_effect = err - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn cls._find_usage_spot_fleets() total = cls.limits['Max active spot fleets per ' 'region'].get_current_usage() assert len(total) == 0 - def test_find_usage_spot_fleets_unknown_code(self): + def test_unknown_code(self): mock_client_conn = Mock() err = botocore.exceptions.ClientError( {'Error': {'Code': 'SomeCode'}}, 'operation', ) mock_client_conn.describe_spot_fleet_requests.side_effect = err - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn with pytest.raises(botocore.exceptions.ClientError): cls._find_usage_spot_fleets() - def test_find_usage_spot_fleets_unknown_error(self): + def test_unknown_error(self): mock_client_conn = Mock() mock_client_conn.describe_spot_fleet_requests.side_effect = RuntimeError - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn with pytest.raises(RuntimeError): cls._find_usage_spot_fleets() - def test_update_limits_from_api(self): + +class TestUpdateLimitsFromApi(object): + + def test_happy_path(self): data = fixtures.test_update_limits_from_api mock_conn = Mock() mock_client_conn = Mock() mock_client_conn.describe_account_attributes.return_value = data - cls = _Ec2Service(21, 43) - cls.resource_conn = mock_conn - cls.conn = mock_client_conn - with patch('awslimitchecker.services.ec2.logger') as mock_logger: - cls._update_limits_from_api() + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + cls = _Ec2Service(21, 43, {}, None) + cls.resource_conn = mock_conn + cls.conn = mock_client_conn + with patch('awslimitchecker.services.ec2.logger') as mock_logger: + cls._update_limits_from_api() assert mock_conn.mock_calls == [] assert mock_client_conn.mock_calls == [ call.describe_account_attributes() @@ -754,14 +1168,373 @@ def test_update_limits_from_api(self): assert cls.limits['VPC security groups per elastic ' 'network interface'].api_limit == 5 - def test_update_limits_from_api_unsupported(self): + def test_vcpu(self): + data = fixtures.test_update_limits_from_api_vcpu + mock_conn = Mock() + mock_client_conn = Mock() + mock_client_conn.describe_account_attributes.return_value = data + + with patch( + '%s._use_vcpu_limits' % pb, new_callable=PropertyMock + ) as m_use_vcpu: + m_use_vcpu.return_value = False + cls = _Ec2Service(21, 43, {}, None) + cls.resource_conn = mock_conn + cls.conn = mock_client_conn + with patch('awslimitchecker.services.ec2.logger') as mock_logger: + cls._update_limits_from_api() + assert mock_conn.mock_calls == [] + assert mock_client_conn.mock_calls == [ + call.describe_account_attributes() + ] + assert mock_logger.mock_calls == [ + call.info("Querying EC2 DescribeAccountAttributes for limits"), + call.debug('Done setting limits from API') + ] + assert cls.limits['Elastic IP addresses (EIPs)'].api_limit == 40 + assert cls.limits['VPC Elastic IP addresses (EIPs)'].api_limit == 200 + assert cls.limits['VPC security groups per elastic ' + 'network interface'].api_limit == 5 + + def test_unsupported(self): data = fixtures.test_update_limits_from_api_unsupported mock_client_conn = Mock() mock_client_conn.describe_account_attributes.return_value = data - cls = _Ec2Service(21, 43) + cls = _Ec2Service(21, 43, {}, None) cls.conn = mock_client_conn cls._update_limits_from_api() lim = cls.limits['Elastic IP addresses (EIPs)'] usage = lim.get_current_usage() assert len(usage) == 0 + + +class TestUseVcpuLimits(object): + + @patch.dict( + os.environ, + {}, + clear=True + ) + def test_useast1(self): + with patch('%s.get_limits' % pb): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-east-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is True + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {}, + clear=True + ) + def test_beijing(self): + with patch('%s.get_limits' % pb): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-north-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {}, + clear=True + ) + def test_ningxia(self): + with patch('%s.get_limits' % pb): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-northwest-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {}, + clear=True + ) + def test_gov_west1(self): + with patch('%s.get_limits' % pb): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-gov-west-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'true'}, + clear=True + ) + def test_useast1_env_true(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-east-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is True + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'true'}, + clear=True + ) + def test_beijing_env_true(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-north-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is True + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'true'}, + clear=True + ) + def test_ningxia_env_true(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-northwest-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is True + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'true'}, + clear=True + ) + def test_gov_west1_env_true(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-gov-west-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is True + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'false'}, + clear=True + ) + def test_useast1_env_false(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-east-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'false'}, + clear=True + ) + def test_beijing_env_false(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-north-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'false'}, + clear=True + ) + def test_ningxia_env_false(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='cn-northwest-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn + + @patch.dict( + os.environ, + {'USE_VCPU_LIMITS': 'false'}, + clear=True + ) + def test_gov_west1_env_false(self): + cls = _Ec2Service(21, 43, {}, None) + mock_orig_conn = Mock() + cls.conn = mock_orig_conn + + def se_conn(klass): + mock_conn = Mock() + mock_conf = Mock() + type(mock_conf).region_name = PropertyMock( + return_value='us-gov-west-1' + ) + type(mock_conn)._client_config = PropertyMock( + return_value=mock_conf + ) + klass.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_conn + res = cls._use_vcpu_limits + assert res is False + assert cls.conn == mock_orig_conn diff --git a/awslimitchecker/tests/services/test_ecs.py b/awslimitchecker/tests/services/test_ecs.py index 97b51047..d0f886ca 100644 --- a/awslimitchecker/tests/services/test_ecs.py +++ b/awslimitchecker/tests/services/test_ecs.py @@ -60,7 +60,7 @@ class Test_EcsService(object): def test_init(self): """test __init__()""" - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) assert cls.service_name == 'ECS' assert cls.api_name == 'ecs' assert cls.conn is None @@ -68,7 +68,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -86,7 +86,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -98,7 +98,7 @@ def test_find_usage(self): connect=DEFAULT, _find_usage_clusters=DEFAULT ) as mocks: - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) assert cls._have_usage is False cls.find_usage() assert mocks['connect'].mock_calls == [call(cls)] @@ -161,7 +161,7 @@ def se_clusters(*_, **kwargs): }] mock_conn.get_paginator.return_value = mock_paginator - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) cls.conn = mock_conn with patch('%s._find_usage_one_cluster' % pb, autospec=True) as m_fuoc: cls._find_usage_clusters() @@ -249,7 +249,7 @@ def se_cluster(*_, **kwargs): }] mock_conn.get_paginator.return_value = mock_paginator - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_one_cluster('cName') @@ -274,7 +274,7 @@ def se_cluster(*_, **kwargs): assert u[1].aws_type == 'AWS::ECS::Service' def test_required_iam_permissions(self): - cls = _EcsService(21, 43) + cls = _EcsService(21, 43, {}, None) assert sorted(cls.required_iam_permissions()) == [ 'ecs:DescribeClusters', 'ecs:DescribeServices', diff --git a/awslimitchecker/tests/services/test_efs.py b/awslimitchecker/tests/services/test_efs.py index 57d8530e..5052a218 100644 --- a/awslimitchecker/tests/services/test_efs.py +++ b/awslimitchecker/tests/services/test_efs.py @@ -61,7 +61,7 @@ class Test_EfsService(object): def test_init(self): """test __init__()""" - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) assert cls.service_name == 'EFS' assert cls.api_name == 'efs' assert cls.conn is None @@ -69,7 +69,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -80,56 +80,10 @@ def test_get_limits(self): assert res['File systems'].def_critical_threshold == 43 assert res['File systems'].default_limit == 1000 - def test_update_limits_from_api(self): - mock_conn = Mock() - mock_conf = Mock() - type(mock_conf).region_name = 'us-west-2' - mock_conn._client_config = mock_conf - with patch('%s.connect' % pb, create=True) as mock_connect: - cls = _EfsService(21, 43) - cls.conn = mock_conn - cls.limits = { - 'File systems': AwsLimit( - 'File systems', - cls, - 1000, - cls.warning_threshold, - cls.critical_threshold, - limit_type='AWS::EFS::FileSystem', - ) - } - cls._update_limits_from_api() - assert mock_connect.mock_calls == [call()] - assert mock_conn.mock_calls == [] - assert cls.limits['File systems'].default_limit == 1000 - - def test_update_limits_from_api_us_east_1(self): - mock_conn = Mock() - mock_conf = Mock() - type(mock_conf).region_name = 'us-east-1' - mock_conn._client_config = mock_conf - with patch('%s.connect' % pb, create=True) as mock_connect: - cls = _EfsService(21, 43) - cls.conn = mock_conn - cls.limits = { - 'File systems': AwsLimit( - 'File systems', - cls, - 1000, - cls.warning_threshold, - cls.critical_threshold, - limit_type='AWS::EFS::FileSystem', - ) - } - cls._update_limits_from_api() - assert mock_connect.mock_calls == [call()] - assert mock_conn.mock_calls == [] - assert cls.limits['File systems'].default_limit == 70 - def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock(spec_set=AwsLimit) - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -145,7 +99,7 @@ def test_find_usage(self): {'FileSystemId': 'baz'} ] } - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -173,7 +127,7 @@ def test_find_usage_no_endpoint(self): with patch('%s.connect' % pb) as mock_connect: with patch('%s.paginate_dict' % pbm) as mock_paginate: mock_paginate.side_effect = exc - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -206,7 +160,7 @@ def test_find_usage_access_denied(self): with patch('%s.connect' % pb) as mock_connect: with patch('%s.paginate_dict' % pbm) as mock_paginate: mock_paginate.side_effect = exc - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -225,7 +179,7 @@ def test_find_usage_access_denied(self): assert len(usage) == 0 def test_required_iam_permissions(self): - cls = _EfsService(21, 43) + cls = _EfsService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'elasticfilesystem:DescribeFileSystems' ] diff --git a/awslimitchecker/tests/services/test_elasticache.py b/awslimitchecker/tests/services/test_elasticache.py index 11c401f0..a4b2a570 100644 --- a/awslimitchecker/tests/services/test_elasticache.py +++ b/awslimitchecker/tests/services/test_elasticache.py @@ -61,14 +61,14 @@ class TestElastiCacheService(object): def test_init(self): """test __init__()""" - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) assert cls.service_name == 'ElastiCache' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -87,7 +87,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -104,7 +104,7 @@ def test_find_usage(self): _find_usage_parameter_groups=DEFAULT, _find_usage_security_groups=DEFAULT, ) as mocks: - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -128,7 +128,7 @@ def test_find_usage_nodes(self): mock_paginator = Mock() mock_paginator.paginate.return_value = responses mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_nodes() @@ -158,7 +158,7 @@ def test_find_usage_subnet_groups(self): mock_paginator = Mock() mock_paginator.paginate.return_value = responses mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_subnet_groups() @@ -189,7 +189,7 @@ def test_find_usage_parameter_groups(self): mock_paginator = Mock() mock_paginator.paginate.return_value = responses mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_parameter_groups() @@ -214,7 +214,7 @@ def test_find_usage_security_groups(self): mock_paginator = Mock() mock_paginator.paginate.return_value = responses mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_security_groups() @@ -253,7 +253,7 @@ def se_exc(*args, **kwargs): mock_paginator.paginate.side_effect = se_exc mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % self.pbm) as mock_logger: @@ -298,7 +298,7 @@ def se_exc(*args, **kwargs): mock_paginator.paginate.side_effect = se_exc mock_conn.get_paginator.return_value = mock_paginator - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) cls.conn = mock_conn with pytest.raises(Exception) as raised: @@ -314,7 +314,7 @@ def se_exc(*args, **kwargs): assert raised.value == exc def test_required_iam_permissions(self): - cls = _ElastiCacheService(21, 43) + cls = _ElastiCacheService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "elasticache:DescribeCacheClusters", "elasticache:DescribeCacheParameterGroups", diff --git a/awslimitchecker/tests/services/test_elasticbeanstalk.py b/awslimitchecker/tests/services/test_elasticbeanstalk.py index 56a6657f..dd3a2668 100644 --- a/awslimitchecker/tests/services/test_elasticbeanstalk.py +++ b/awslimitchecker/tests/services/test_elasticbeanstalk.py @@ -60,7 +60,7 @@ class Test_ElasticBeanstalkService(object): def test_init(self): """test __init__()""" - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) assert cls.service_name == 'ElasticBeanstalk' assert cls.api_name == 'elasticbeanstalk' assert cls.conn is None @@ -69,7 +69,7 @@ def test_init(self): def test_get_limits(self): """test get limits returns all keys""" - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -85,7 +85,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -100,7 +100,7 @@ def test_find_usage(self): _find_usage_application_versions=DEFAULT, _find_usage_environments=DEFAULT, ) as mocks: - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -120,7 +120,7 @@ def test_find_usage_applications(self): mock_conn = Mock() mock_conn.describe_applications.return_value = response - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_applications() @@ -139,7 +139,7 @@ def test_find_usage_application_versions(self): mock_conn = Mock() mock_conn.describe_application_versions.return_value = response - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_application_versions() @@ -157,7 +157,7 @@ def test_find_usage_environments(self): mock_conn = Mock() mock_conn.describe_environments.return_value = response - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_environments() @@ -171,7 +171,7 @@ def test_find_usage_environments(self): def test_required_iam_permissions(self): """test expected permissions are returned""" - cls = _ElasticBeanstalkService(21, 43) + cls = _ElasticBeanstalkService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'elasticbeanstalk:DescribeApplications', 'elasticbeanstalk:DescribeApplicationVersions', diff --git a/awslimitchecker/tests/services/test_elb.py b/awslimitchecker/tests/services/test_elb.py index 076248ca..a18ad7c4 100644 --- a/awslimitchecker/tests/services/test_elb.py +++ b/awslimitchecker/tests/services/test_elb.py @@ -60,14 +60,14 @@ class Test_ElbService(object): def test_init(self): """test __init__()""" - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) assert cls.service_name == 'ELB' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -90,7 +90,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -109,7 +109,7 @@ def test_update_limits_from_api(self): return_value='rname' ) m_cli.describe_account_limits.return_value = r2 - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls.conn = mock_conn cls._boto3_connection_kwargs = {'foo': 'bar', 'baz': 'blam'} cls.get_limits() @@ -138,7 +138,7 @@ def test_find_usage(self): with patch('%s._find_usage_elbv2' % pb, autospec=True) as mock_v2: mock_v1.return_value = 3 mock_v2.return_value = 5 - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) assert cls._have_usage is False cls.find_usage() assert cls._have_usage is True @@ -161,7 +161,7 @@ def test_find_usage_elbv1(self): with patch('%s.connect' % pb) as mock_connect: with patch('%s.paginate_dict' % pbm) as mock_paginate: mock_paginate.return_value = return_value - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls.conn = mock_conn res = cls._find_usage_elbv1() assert res == 4 @@ -217,7 +217,7 @@ def test_find_usage_elbv2(self): tgs_res, lbs_res ] - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls._boto3_connection_kwargs = { 'foo': 'bar', 'baz': 'blam' @@ -268,7 +268,7 @@ def test_update_usage_for_alb(self): result_fixtures.ELB.test_usage_alb_rules[1], result_fixtures.ELB.test_usage_alb_rules[2] ] - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls._update_usage_for_alb(conn, 'myarn', 'albname') assert mock_paginate.mock_calls == [ call( @@ -326,7 +326,7 @@ def test_update_usage_for_nlb(self): mock_paginate.side_effect = [ result_fixtures.ELB.test_usage_nlb_listeners ] - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) cls._update_usage_for_nlb(conn, 'mynarn', 'nlbname') assert mock_paginate.mock_calls == [ call( @@ -346,7 +346,7 @@ def test_update_usage_for_nlb(self): assert lim[0].resource_id == 'nlbname' def test_required_iam_permissions(self): - cls = _ElbService(21, 43) + cls = _ElbService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "elasticloadbalancing:DescribeLoadBalancers", "elasticloadbalancing:DescribeAccountLimits", diff --git a/awslimitchecker/tests/services/test_firehose.py b/awslimitchecker/tests/services/test_firehose.py index 203f82b7..ed06b92d 100644 --- a/awslimitchecker/tests/services/test_firehose.py +++ b/awslimitchecker/tests/services/test_firehose.py @@ -63,7 +63,7 @@ class Test_FirehoseService(object): def test_init(self): """test __init__()""" - cls = _FirehoseService(21, 43) + cls = _FirehoseService(21, 43, {}, None) assert cls.service_name == 'Firehose' assert cls.api_name == 'firehose' assert cls.conn is None @@ -71,7 +71,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _FirehoseService(21, 43) + cls = _FirehoseService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -85,7 +85,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _FirehoseService(21, 43) + cls = _FirehoseService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -99,7 +99,7 @@ def test_find_usage(self): response_streams.append(stream_name) mock_conn = Mock() mock_conn.list_delivery_streams.side_effect = responses - cls = _FirehoseService(21, 43, {'region_name': 'us-west-2'}) + cls = _FirehoseService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn cls.find_usage() assert mock_conn.list_delivery_streams.call_count == len(responses) @@ -111,7 +111,7 @@ def test_find_usage(self): assert usage[0].aws_type == 'AWS::KinesisFirehose::DeliveryStream' def test_required_iam_permissions(self): - cls = _FirehoseService(21, 43) + cls = _FirehoseService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "firehose:ListDeliveryStreams", ] @@ -121,7 +121,7 @@ def test_find_usage_with_endpoint_connection_error(self): client_error = EndpointConnectionError( endpoint_url='https://firehose.bad-region.amazonaws.com/') mock_conn.list_delivery_streams.side_effect = client_error - cls = _FirehoseService(21, 43) + cls = _FirehoseService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % self.pbm, autospec=True) as mock_logger: cls.find_usage() diff --git a/awslimitchecker/tests/services/test_iam.py b/awslimitchecker/tests/services/test_iam.py index 1cccc822..60b096bf 100644 --- a/awslimitchecker/tests/services/test_iam.py +++ b/awslimitchecker/tests/services/test_iam.py @@ -60,7 +60,7 @@ class Test_IamService(object): def test_init(self): """test __init__()""" - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) assert cls.service_name == 'IAM' assert cls.api_name == 'iam' assert cls.conn is None @@ -68,7 +68,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -88,21 +88,21 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits def test_find_usage(self): with patch('%s._update_limits_from_api' % pb) as mock_update: - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) assert cls._have_usage is False cls.find_usage() assert mock_update.mock_calls == [call()] assert cls._have_usage is True def test_required_iam_permissions(self): - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'iam:GetAccountSummary' ] @@ -115,7 +115,7 @@ def test_update_limits_from_api(self): mock_conn.AccountSummary.return_value = mock_summary with patch('%s.logger' % pbm) as mock_logger: with patch('%s.connect_resource' % pb) as mock_connect: - cls = _IamService(21, 43) + cls = _IamService(21, 43, {}, None) cls.resource_conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] diff --git a/awslimitchecker/tests/services/test_lambdafunc.py b/awslimitchecker/tests/services/test_lambdafunc.py index 5c1ab56a..ff083438 100644 --- a/awslimitchecker/tests/services/test_lambdafunc.py +++ b/awslimitchecker/tests/services/test_lambdafunc.py @@ -61,7 +61,7 @@ class Test_LambdaService(object): def test_init(self): """test __init__()""" - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) assert cls.service_name == 'Lambda' assert cls.api_name == 'lambda' assert cls.conn is None @@ -69,7 +69,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == [ @@ -88,7 +88,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -99,7 +99,7 @@ def test_update_limits_from_api(self): mock_conn.get_account_settings.return_value = response with patch('%s.connect' % pb) as mock_connect: - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) assert len(cls.limits) == 6 cls.conn = mock_conn cls._update_limits_from_api() @@ -124,7 +124,7 @@ def test_update_limits_from_api_exit_early(self): mock_conn.get_account_settings.return_value = response with patch('%s.connect' % pb) as mock_connect: - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) assert len(cls.limits) == 6 cls.limits = {"a": None, "b": None} cls.conn = mock_conn @@ -140,7 +140,7 @@ def conn_err(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) cls.conn = mock_conn cls.find_usage() @@ -160,7 +160,7 @@ def test_find_usage(self): mock_conn.get_account_settings.return_value = response with patch('%s.connect' % pb) as mock_connect: - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.get_limits() @@ -178,7 +178,7 @@ def test_find_usage(self): assert u[0].get_value() == 2 def test_required_iam_permissions(self): - cls = _LambdaService(21, 43) + cls = _LambdaService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'lambda:GetAccountSettings' ] diff --git a/awslimitchecker/tests/services/test_rds.py b/awslimitchecker/tests/services/test_rds.py index 650ee6da..8d419119 100644 --- a/awslimitchecker/tests/services/test_rds.py +++ b/awslimitchecker/tests/services/test_rds.py @@ -59,14 +59,14 @@ class Test_RDSService(object): def test_init(self): """test __init__()""" - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) assert cls.service_name == 'RDS' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -96,7 +96,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -112,7 +112,7 @@ def test_find_usage(self): _find_usage_security_groups=DEFAULT, _update_limits_from_api=DEFAULT, ) as mocks: - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -127,7 +127,7 @@ def test_find_usage(self): assert mocks[x].mock_calls == [call()] def test_required_iam_permissions(self): - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "rds:DescribeAccountAttributes", "rds:DescribeDBInstances", @@ -148,7 +148,7 @@ def test_find_usage_instances(self): mock_paginator.paginate.return_value = instances mock_conn.get_paginator.return_value = mock_paginator - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_instances() @@ -178,7 +178,7 @@ def test_find_usage_subnet_groups(self): mock_paginator.paginate.return_value = data mock_conn.get_paginator.return_value = mock_paginator - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_subnet_groups() @@ -213,7 +213,7 @@ def test_find_usage_security_groups(self): mock_paginator.paginate.return_value = data mock_conn.get_paginator.return_value = mock_paginator - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_security_groups() @@ -257,7 +257,7 @@ def test_update_limits_from_api(self): mock_conn.describe_account_attributes.return_value = response with patch('%s.logger' % self.pbm) as mock_logger: with patch('%s.connect' % self.pb) as mock_connect: - cls = _RDSService(21, 43) + cls = _RDSService(21, 43, {}, None) cls.conn = mock_conn # limits that we still calculate usage for cls.limits['Max auths per security group']._add_current_usage(1) diff --git a/awslimitchecker/tests/services/test_redshift.py b/awslimitchecker/tests/services/test_redshift.py index 9ace2d6f..14b48797 100644 --- a/awslimitchecker/tests/services/test_redshift.py +++ b/awslimitchecker/tests/services/test_redshift.py @@ -60,7 +60,7 @@ class Test_RedshiftService(object): def test_init(self): """test __init__()""" - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {}, None) assert cls.service_name == 'Redshift' assert cls.api_name == 'redshift' assert cls.conn is None @@ -68,7 +68,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -83,7 +83,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -97,7 +97,7 @@ def test_find_usage(self): _find_cluster_manual_snapshots=DEFAULT, _find_cluster_subnet_groups=DEFAULT, ) as mocks: - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -117,7 +117,7 @@ def test_find_usage_manual_snapshots(self): mock_conn = Mock() mock_conn.describe_cluster_snapshots.return_value = response - cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}) + cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn cls._find_cluster_manual_snapshots() @@ -135,7 +135,7 @@ def test_find_usage_subnet_groups(self): mock_conn = Mock() mock_conn.describe_cluster_subnet_groups.return_value = response - cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}) + cls = _RedshiftService(21, 43, {'region_name': 'us-west-2'}, None) cls.conn = mock_conn cls._find_cluster_subnet_groups() @@ -145,7 +145,7 @@ def test_find_usage_subnet_groups(self): 0].get_value() == 3 def test_required_iam_permissions(self): - cls = _RedshiftService(21, 43) + cls = _RedshiftService(21, 43, {}, None) assert cls.required_iam_permissions() == [ "redshift:DescribeClusterSnapshots", "redshift:DescribeClusterSubnetGroups", diff --git a/awslimitchecker/tests/services/test_route53.py b/awslimitchecker/tests/services/test_route53.py index 94136dfb..2fc80f1b 100644 --- a/awslimitchecker/tests/services/test_route53.py +++ b/awslimitchecker/tests/services/test_route53.py @@ -70,7 +70,7 @@ def _mock_reponse_init(self, cls): def test_init(self): """test __init__()""" - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) assert cls.service_name == 'Route53' assert cls.api_name == 'route53' assert cls.conn is None @@ -81,7 +81,7 @@ def test_get_limits(self): """ Test that get_limits returns AwsLimits """ - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -94,7 +94,7 @@ def test_get_limits(self): assert limit.def_critical_threshold == 43 def test_find_usage(self): - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) assert cls._have_usage is False cls.find_usage() @@ -109,7 +109,7 @@ def test_update_limits_from_api(self): pb, _find_limit_hosted_zone=DEFAULT, ) as mocks: - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -120,7 +120,7 @@ def test_update_limits_from_api(self): assert mocks[x].mock_calls == [call()] def test_find_limit_hosted_zone_recordsets(self): - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) self._mock_reponse_init(cls) cls._find_limit_hosted_zone() @@ -143,7 +143,7 @@ def test_find_limit_hosted_zone_recordsets(self): assert usage2.get_maximum() == 10002 def test_find_limit_hosted_zone_vpc_associations(self): - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) self._mock_reponse_init(cls) cls._find_limit_hosted_zone() @@ -161,7 +161,7 @@ def test_find_limit_hosted_zone_vpc_associations(self): assert usage2.get_maximum() == 101 def test_required_iam_permissions(self): - cls = _Route53Service(21, 43) + cls = _Route53Service(21, 43, {}, None) assert cls.required_iam_permissions() == [ "route53:GetHostedZone", "route53:GetHostedZoneLimit", diff --git a/awslimitchecker/tests/services/test_s3.py b/awslimitchecker/tests/services/test_s3.py index aa6e7212..aaa59316 100644 --- a/awslimitchecker/tests/services/test_s3.py +++ b/awslimitchecker/tests/services/test_s3.py @@ -60,7 +60,7 @@ class Test_S3Service(object): def test_init(self): """test __init__()""" - cls = _S3Service(21, 43) + cls = _S3Service(21, 43, {}, None) assert cls.service_name == 'S3' assert cls.api_name == 's3' assert cls.conn is None @@ -68,7 +68,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _S3Service(21, 43) + cls = _S3Service(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -82,7 +82,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock(spec_set=AwsLimit) - cls = _S3Service(21, 43) + cls = _S3Service(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -92,7 +92,7 @@ def test_find_usage(self): mock_buckets.all.return_value = ['a', 'b', 'c'] mock_conn = Mock(buckets=mock_buckets) with patch('%s.connect_resource' % pb) as mock_connect: - cls = _S3Service(21, 43) + cls = _S3Service(21, 43, {}, None) cls.resource_conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -103,7 +103,7 @@ def test_find_usage(self): assert cls.limits['Buckets'].get_current_usage()[0].get_value() == 3 def test_required_iam_permissions(self): - cls = _S3Service(21, 43) + cls = _S3Service(21, 43, {}, None) assert cls.required_iam_permissions() == [ 's3:ListAllMyBuckets' ] diff --git a/awslimitchecker/tests/services/test_ses.py b/awslimitchecker/tests/services/test_ses.py index ea9c4b31..619a736d 100644 --- a/awslimitchecker/tests/services/test_ses.py +++ b/awslimitchecker/tests/services/test_ses.py @@ -61,7 +61,7 @@ class Test_SesService(object): def test_init(self): """test __init__()""" - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) assert cls.service_name == 'SES' assert cls.api_name == 'ses' assert cls.conn is None @@ -69,7 +69,7 @@ def test_init(self): assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -84,7 +84,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -98,7 +98,7 @@ def test_find_usage(self): } with patch('%s.connect' % pb) as mock_connect: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -118,7 +118,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -156,7 +156,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -192,7 +192,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -229,7 +229,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False with pytest.raises(ClientError): @@ -251,7 +251,7 @@ def test_update_limits_from_api(self): } with patch('%s.connect' % pb) as mock_connect: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -267,7 +267,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -302,7 +302,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -333,7 +333,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn cls._update_limits_from_api() assert mock_connect.mock_calls == [call()] @@ -365,7 +365,7 @@ def se_get(): with patch('%s.connect' % pb) as mock_connect: with patch('%s.logger' % pbm) as mock_logger: - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) cls.conn = mock_conn with pytest.raises(ClientError): cls._update_limits_from_api() @@ -375,7 +375,7 @@ def se_get(): assert cls.limits['Daily sending quota'].api_limit is None def test_required_iam_permissions(self): - cls = _SesService(21, 43) + cls = _SesService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'ses:GetSendQuota' ] diff --git a/awslimitchecker/tests/services/test_vpc.py b/awslimitchecker/tests/services/test_vpc.py index a461c87b..ed651035 100644 --- a/awslimitchecker/tests/services/test_vpc.py +++ b/awslimitchecker/tests/services/test_vpc.py @@ -61,14 +61,14 @@ class Test_VpcService(object): def test_init(self): """test __init__()""" - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) assert cls.service_name == 'VPC' assert cls.conn is None assert cls.warning_threshold == 21 assert cls.critical_threshold == 43 def test_get_limits(self): - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.limits = {} res = cls.get_limits() assert sorted(res.keys()) == sorted([ @@ -91,7 +91,7 @@ def test_get_limits(self): def test_get_limits_again(self): """test that existing limits dict is returned on subsequent calls""" mock_limits = Mock() - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.limits = mock_limits res = cls.get_limits() assert res == mock_limits @@ -116,7 +116,7 @@ def test_find_usage(self): _find_usage_network_interfaces=DEFAULT, ) as mocks: mocks['_find_usage_subnets'].return_value = sn - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn assert cls._have_usage is False cls.find_usage() @@ -141,7 +141,7 @@ def test_find_usage_vpcs(self): mock_conn = Mock() mock_conn.describe_vpcs.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_vpcs() @@ -157,7 +157,7 @@ def test_find_usage_subnets(self): mock_conn = Mock() mock_conn.describe_subnets.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn res = cls._find_usage_subnets() @@ -181,7 +181,7 @@ def test_find_usage_acls(self): response = result_fixtures.VPC.test_find_usage_acls mock_conn = Mock() - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn mock_conn.describe_network_acls.return_value = response @@ -212,7 +212,7 @@ def test_find_usage_route_tables(self): mock_conn = Mock() mock_conn.describe_route_tables.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_route_tables() @@ -242,7 +242,7 @@ def test_find_usage_internet_gateways(self): mock_conn = Mock() mock_conn.describe_internet_gateways.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_gateways() @@ -262,7 +262,7 @@ def test_find_usage_nat_gateways(self): mock_conn.describe_nat_gateways.return_value = response with patch('%s.logger' % self.pbm) as mock_logger: - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_nat_gateways(subnets) @@ -299,7 +299,7 @@ def se_exc(*args, **kwargs): mock_conn = Mock() mock_conn.describe_nat_gateways.side_effect = se_exc - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn with patch('%s.logger' % self.pbm, autospec=True) as mock_logger: @@ -321,7 +321,7 @@ def test_find_usages_vpn_gateways(self): mock_conn = Mock() mock_conn.describe_vpn_gateways.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usages_vpn_gateways() @@ -348,7 +348,7 @@ def test_find_usage_network_interfaces(self): mock_conn = Mock() mock_conn.describe_network_interfaces.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.conn = mock_conn cls._find_usage_network_interfaces() @@ -369,7 +369,7 @@ def test_update_limits_from_api_high_max_instances(self): mock_client_conn = Mock() mock_client_conn.describe_account_attributes.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.resource_conn = mock_conn cls.conn = mock_client_conn with patch('awslimitchecker.services.vpc.logger') as mock_logger: @@ -393,7 +393,7 @@ def test_update_limits_from_api_low_max_instances(self): mock_client_conn = Mock() mock_client_conn.describe_account_attributes.return_value = response - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) cls.resource_conn = mock_conn cls.conn = mock_client_conn with patch('awslimitchecker.services.vpc.logger') as mock_logger: @@ -412,7 +412,7 @@ def test_update_limits_from_api_low_max_instances(self): assert cls.limits[limit_name].get_limit() == DEFAULT_ENI_LIMIT def test_required_iam_permissions(self): - cls = _VpcService(21, 43) + cls = _VpcService(21, 43, {}, None) assert cls.required_iam_permissions() == [ 'ec2:DescribeNatGateways', 'ec2:DescribeNetworkAcls', diff --git a/awslimitchecker/tests/support.py b/awslimitchecker/tests/support.py index bb013338..d6e06352 100644 --- a/awslimitchecker/tests/support.py +++ b/awslimitchecker/tests/support.py @@ -155,6 +155,10 @@ def unexpected_logs(self, allow_endpoint_error=False): r.funcName == 'find_usage' and 'perhaps the Firehose ' 'service is not available in this region' in r.msg): continue + if (r.levelno == logging.WARNING and r.module == 'quotas' and + r.funcName == 'quotas_for_service' and + 'Attempted to retrieve Service Quotas' in r.msg): + continue res.append('%s:%s.%s (%s:%s) %s - %s %s' % ( r.name, r.module, @@ -307,12 +311,22 @@ def sample_limits_api(): limit_type='ltfoo5', limit_subtype='sltfoo5', ), + 'zzz limit5': AwsLimit( + 'zzz limit5', + 'SvcFoo', + 4, + 1, + 5, + limit_type='ltfoo5', + limit_subtype='sltfoo5', + ), }, } limits['SvcBar']['bar limit2']._set_api_limit(2) limits['SvcBar']['bar limit2'].set_limit_override(99) limits['SvcFoo']['foo limit3']._set_ta_limit(10) limits['SvcFoo']['zzz limit4']._set_api_limit(34) + limits['SvcFoo']['zzz limit5']._set_quotas_limit(60.0) limits['SvcFoo']['limit with usage maximums']._add_current_usage( 1, @@ -320,3 +334,55 @@ def sample_limits_api(): aws_type='res_type', resource_id='res_id') return limits + + +def quotas_response(): + return ( + [ + { + 'Quotas': [ + { + 'QuotaName': 'qname1', + 'QuotaCode': 'qcode1', + 'Value': 1.1 + }, + { + 'QuotaName': 'qname2', + 'QuotaCode': 'qcode2', + 'Value': 2.2 + } + ] + }, + { + 'Quotas': [ + { + 'QuotaName': 'qname3', + 'QuotaCode': 'qcode3', + 'Value': 3.3 + }, + { + 'QuotaName': 'qname2', + 'QuotaCode': 'qcode2', + 'Value': 2.4 # triggers the error log for dupe + } + ] + } + ], + { + 'qname1': { + 'QuotaName': 'qname1', + 'QuotaCode': 'qcode1', + 'Value': 1.1 + }, + 'qname2': { + 'QuotaName': 'qname2', + 'QuotaCode': 'qcode2', + 'Value': 2.4 + }, + 'qname3': { + 'QuotaName': 'qname3', + 'QuotaCode': 'qcode3', + 'Value': 3.3 + } + } + ) diff --git a/awslimitchecker/tests/test_checker.py b/awslimitchecker/tests/test_checker.py index af12b632..4c613716 100644 --- a/awslimitchecker/tests/test_checker.py +++ b/awslimitchecker/tests/test_checker.py @@ -96,12 +96,14 @@ def setup(self): _get_version_info=DEFAULT, TrustedAdvisor=DEFAULT, _get_latest_version=DEFAULT, + ServiceQuotasClient=DEFAULT, autospec=True, ) as mocks: self.mock_logger = mocks['logger'] self.mock_version = mocks['_get_version_info'] self.mock_ta_constr = mocks['TrustedAdvisor'] self.mock_glv = mocks['_get_latest_version'] + self.mock_quotas = mocks['ServiceQuotasClient'] mocks['TrustedAdvisor'].return_value = self.mock_ta mocks['_get_latest_version'].return_value = None self.mock_version.return_value = self.mock_ver_info @@ -116,13 +118,13 @@ def test_init(self): assert self.cls.services == services # _AwsService instances should exist, but have no other calls assert self.mock_foo.mock_calls == [ - call(80, 99, {'region_name': None}) + call(80, 99, {'region_name': None}, self.mock_quotas.return_value) ] assert self.mock_bar.mock_calls == [ - call(80, 99, {'region_name': None}) + call(80, 99, {'region_name': None}, self.mock_quotas.return_value) ] assert self.mock_ta_constr.mock_calls == [ - call(services, {'region_name': None}, + call(services, {'region_name': None}, ta_api_region='us-east-1', ta_refresh_mode=None, ta_refresh_timeout=None) ] assert self.mock_svc1.mock_calls == [] @@ -134,6 +136,10 @@ def test_init(self): assert self.mock_logger.mock_calls == [ call.debug('Connecting to region %s', None) ] + assert self.cls.role_partition == 'aws' + assert self.mock_quotas.mock_calls == [ + call({'region_name': None}) + ] def test_init_AGPL_message(self, capsys): # get rid of the class @@ -204,6 +210,7 @@ def test_init_thresholds(self): _get_version_info=DEFAULT, TrustedAdvisor=DEFAULT, _get_latest_version=DEFAULT, + ServiceQuotasClient=DEFAULT, autospec=True, ) as mocks: mock_version = mocks['_get_version_info'] @@ -223,13 +230,19 @@ def test_init_thresholds(self): assert cls.services == services # _AwsService instances should exist, but have no other calls assert mock_foo.mock_calls == [ - call(5, 22, {'region_name': None}) + call( + 5, 22, {'region_name': None}, + mocks['ServiceQuotasClient'].return_value + ) ] assert mock_bar.mock_calls == [ - call(5, 22, {'region_name': None}) + call( + 5, 22, {'region_name': None}, + mocks['ServiceQuotasClient'].return_value + ) ] assert mock_ta_constr.mock_calls == [ - call(services, {'region_name': None}, + call(services, {'region_name': None}, ta_api_region='us-east-1', ta_refresh_mode=None, ta_refresh_timeout=None) ] assert mock_svc1.mock_calls == [] @@ -237,7 +250,7 @@ def test_init_thresholds(self): assert self.mock_version.mock_calls == [call()] assert self.cls.vinfo == self.mock_ver_info - def test_init_region_profile(self): + def test_init_region_profile_role_partition_ta_region(self): mock_svc1 = Mock(spec_set=_AwsService) mock_svc2 = Mock(spec_set=_AwsService) mock_foo = Mock(spec_set=_AwsService) @@ -261,7 +274,15 @@ def test_init_region_profile(self): mock_version.return_value = self.mock_ver_info mocks['TrustedAdvisor'].return_value = mock_ta mocks['_get_latest_version'].return_value = None - cls = AwsLimitChecker(region='regionX', profile_name='foo') + with patch( + '%s._boto_conn_kwargs' % pb, new_callable=PropertyMock + ) as m_bck: + m_bck.return_value = {'region_name': 'rName'} + cls = AwsLimitChecker( + region='regionX', profile_name='foo', + role_partition='rpName', + ta_api_region='taRegion' + ) # dict should be of _AwsService instances services = { 'SvcFoo': mock_svc1, @@ -270,10 +291,20 @@ def test_init_region_profile(self): assert cls.profile_name == 'foo' assert cls.region == 'regionX' assert cls.services == services + assert cls.role_partition == 'rpName' assert mock_svc1.mock_calls == [] assert mock_svc2.mock_calls == [] assert self.mock_version.mock_calls == [call()] assert self.cls.vinfo == self.mock_ver_info + assert mocks['TrustedAdvisor'].mock_calls == [ + call( + services, + {'region_name': 'rName'}, + ta_api_region='taRegion', + ta_refresh_mode=None, + ta_refresh_timeout=None + ) + ] def test_init_sts(self): mock_svc1 = Mock(spec_set=_AwsService) @@ -284,7 +315,19 @@ def test_init_sts(self): mock_foo.return_value = mock_svc1 mock_bar.return_value = mock_svc2 svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar} - with patch('%s.boto3' % pbm): + with patch('%s.boto3' % pbm) as mock_boto: + mock_boto.client.return_value.assume_role.return_value = { + 'Credentials': { + 'AccessKeyId': 'akid', + 'SecretAccessKey': 'sk', + 'SessionToken': 'stoken', + 'Expiration': '0' + }, + 'AssumedRoleUser': { + 'AssumedRoleId': 'arid', + 'Arn': 'arn' + } + } with patch.dict('%s._services' % pbm, values=svcs, clear=True): with patch.multiple( 'awslimitchecker.checker', @@ -313,6 +356,13 @@ def test_init_sts(self): assert mock_svc2.mock_calls == [] assert self.mock_version.mock_calls == [call()] assert self.cls.vinfo == self.mock_ver_info + assert mock_boto.mock_calls == [ + call.client('sts', region_name='myregion'), + call.client().assume_role( + RoleArn='arn:aws:iam::123456789012:role/myrole', + RoleSessionName='awslimitchecker' + ) + ] def test_init_sts_external_id_ta_refresh(self): mock_svc1 = Mock(spec_set=_AwsService) @@ -323,7 +373,19 @@ def test_init_sts_external_id_ta_refresh(self): mock_foo.return_value = mock_svc1 mock_bar.return_value = mock_svc2 svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar} - with patch('%s.boto3' % pbm): + with patch('%s.boto3' % pbm) as mock_boto: + mock_boto.client.return_value.assume_role.return_value = { + 'Credentials': { + 'AccessKeyId': 'akid', + 'SecretAccessKey': 'sk', + 'SessionToken': 'stoken', + 'Expiration': '0' + }, + 'AssumedRoleUser': { + 'AssumedRoleId': 'arid', + 'Arn': 'arn' + } + } with patch.dict('%s._services' % pbm, values=svcs, clear=True): with patch.multiple( 'awslimitchecker.checker', @@ -345,7 +407,8 @@ def test_init_sts_external_id_ta_refresh(self): mfa_serial_number=123, mfa_token=456, ta_refresh_mode=123, - ta_refresh_timeout=456 + ta_refresh_timeout=456, + role_partition='mypart' ) # dict should be of _AwsService instances services = { @@ -357,6 +420,16 @@ def test_init_sts_external_id_ta_refresh(self): assert mock_svc2.mock_calls == [] assert self.mock_version.mock_calls == [call()] assert self.cls.vinfo == self.mock_ver_info + assert mock_boto.mock_calls == [ + call.client('sts', region_name='myregion'), + call.client().assume_role( + ExternalId='myextid', + RoleArn='arn:mypart:iam::123456789012:role/myrole', + RoleSessionName='awslimitchecker', + SerialNumber=123, + TokenCode=456 + ) + ] def test_boto3_connection_kwargs(self): cls = AwsLimitChecker() @@ -375,12 +448,6 @@ def test_boto3_connection_kwargs(self): } def test_boto3_connection_kwargs_profile(self): - with patch('%s.boto3' % pbm): - with patch( - 'awslimitchecker.services.dynamodb._DynamodbService' - '.get_limits' - ): - cls = AwsLimitChecker(profile_name='myprof') m_creds = Mock() type(m_creds).access_key = 'ak' type(m_creds).secret_key = 'sk' @@ -393,8 +460,13 @@ def test_boto3_connection_kwargs_profile(self): with patch('%s._get_sts_token' % pb) as mock_get_sts: with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.Session' % pbm) as mock_sess: - mock_sess.return_value = mock_session - res = cls._boto_conn_kwargs + with patch.dict('%s._services' % pbm, {}, clear=True): + mock_sess.return_value = mock_session + cls = AwsLimitChecker(profile_name='myprof') + mock_get_sts.reset_mock() + mock_logger.reset_mock() + mock_sess.reset_mock() + res = cls._boto_conn_kwargs assert mock_get_sts.mock_calls == [] assert mock_logger.mock_calls == [ call.debug('Using credentials profile: %s', 'myprof') @@ -425,14 +497,6 @@ def test_boto3_connection_kwargs_region(self): } def test_boto3_connection_kwargs_sts(self): - with patch('%s.boto3' % pbm): - with patch( - 'awslimitchecker.services.dynamodb._DynamodbService' - '.get_limits' - ): - cls = AwsLimitChecker(account_id='123', - account_role='myrole', - region='myregion') mock_creds = Mock() type(mock_creds).access_key = 'sts_ak' type(mock_creds).secret_key = 'sts_sk' @@ -441,8 +505,15 @@ def test_boto3_connection_kwargs_sts(self): with patch('%s._get_sts_token' % pb) as mock_get_sts: with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.Session' % pbm) as mock_sess: - mock_get_sts.return_value = mock_creds - res = cls._boto_conn_kwargs + with patch.dict('%s._services' % pbm, {}, clear=True): + cls = AwsLimitChecker(account_id='123', + account_role='myrole', + region='myregion') + mock_get_sts.return_value = mock_creds + mock_get_sts.reset_mock() + mock_logger.reset_mock() + mock_sess.reset_mock() + res = cls._boto_conn_kwargs assert mock_get_sts.mock_calls == [call()] assert mock_logger.mock_calls == [ call.debug("Connecting for account %s role '%s' with STS " @@ -503,10 +574,12 @@ def test_get_limits(self): call.update_limits() ] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.get_limits() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.get_limits() ] @@ -518,10 +591,12 @@ def test_get_limits_no_ta(self): assert res == limits assert self.mock_ta.mock_calls == [] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.get_limits() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.get_limits() ] @@ -535,6 +610,7 @@ def test_get_limits_service(self): call.update_limits() ] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.get_limits() ] assert self.mock_svc2.mock_calls == [] @@ -551,16 +627,19 @@ def test_get_limits_service_with_api(self): assert self.mock_svc1.mock_calls == [] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.get_limits() ] def test_find_usage(self): self.cls.find_usage() assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.find_usage() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.find_usage() ] assert self.mock_ta.mock_calls == [ @@ -570,10 +649,12 @@ def test_find_usage(self): def test_find_usage_no_ta(self): self.cls.find_usage(use_ta=False) assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.find_usage() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.find_usage() ] assert self.mock_ta.mock_calls == [] @@ -581,6 +662,7 @@ def test_find_usage_no_ta(self): def test_find_usage_service(self): self.cls.find_usage(service=['SvcFoo']) assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.find_usage() ] assert self.mock_svc2.mock_calls == [] @@ -593,6 +675,7 @@ def test_find_usage_service_with_api(self): assert self.mock_svc1.mock_calls == [] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.find_usage() ] assert self.mock_ta.mock_calls == [ @@ -767,6 +850,7 @@ def test_get_required_iam_policy(self): 'ec2:foo', 'foo:perm1', 'foo:perm2', + 'servicequotas:ListServiceQuotas', 'support:*', 'trustedadvisor:Describe*', 'trustedadvisor:RefreshCheck' @@ -804,10 +888,12 @@ def test_check_thresholds(self): call.update_limits(), ] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.check_thresholds() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.check_thresholds() ] @@ -824,6 +910,7 @@ def test_check_thresholds_service(self): call.update_limits() ] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.check_thresholds() ] assert self.mock_svc2.mock_calls == [] @@ -843,6 +930,7 @@ def test_check_thresholds_service_api(self): assert self.mock_svc1.mock_calls == [] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.check_thresholds() ] @@ -862,10 +950,12 @@ def test_check_thresholds_no_ta(self): } assert self.mock_ta.mock_calls == [] assert self.mock_svc1.mock_calls == [ + call._update_service_quotas(), call.check_thresholds() ] assert self.mock_svc2.mock_calls == [ call._update_limits_from_api(), + call._update_service_quotas(), call.check_thresholds() ] diff --git a/awslimitchecker/tests/test_connectable.py b/awslimitchecker/tests/test_connectable.py index 36432fd3..009aee2b 100644 --- a/awslimitchecker/tests/test_connectable.py +++ b/awslimitchecker/tests/test_connectable.py @@ -40,6 +40,7 @@ from awslimitchecker.connectable import Connectable, ConnectableCredentials from datetime import datetime import sys +import os # https://code.google.com/p/mock/issues/detail?id=249 # py>=3.4 should use unittest.mock not the mock package on pypi @@ -75,6 +76,54 @@ def __init__(self, account_id=None, account_role=None, region=None, self.profile_name = profile_name +class TestMaxRetriesConfig(object): + + @patch.dict( + os.environ, + {'BOTO_MAX_RETRIES_myapi': '10'}, + clear=True + ) + def test_happy_path(self): + mock_conf = Mock() + cls = ConnectableTester() + cls.api_name = 'myapi' + with patch('%s.Config' % pbm) as m_conf: + m_conf.return_value = mock_conf + res = cls._max_retries_config + assert res == mock_conf + assert m_conf.mock_calls == [call(retries={'max_attempts': 10})] + + @patch.dict( + os.environ, + {}, + clear=True + ) + def test_env_var_not_set(self): + mock_conf = Mock() + cls = ConnectableTester() + cls.api_name = 'myapi' + with patch('%s.Config' % pbm) as m_conf: + m_conf.return_value = mock_conf + res = cls._max_retries_config + assert res is None + assert m_conf.mock_calls == [] + + @patch.dict( + os.environ, + {'BOTO_MAX_RETRIES_myapi': 'hello'}, + clear=True + ) + def test_cant_parse_int(self): + mock_conf = Mock() + cls = ConnectableTester() + cls.api_name = 'myapi' + with patch('%s.Config' % pbm) as m_conf: + m_conf.return_value = mock_conf + res = cls._max_retries_config + assert res is None + assert m_conf.mock_calls == [] + + class Test_Connectable(object): def test_connect(self): @@ -92,8 +141,12 @@ def test_connect(self): mock_kwargs.return_value = kwargs with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.client' % pbm) as mock_client: - mock_client.return_value = mock_conn - cls.connect() + with patch( + '%s._max_retries_config' % pb, new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = None + mock_client.return_value = mock_conn + cls.connect() assert mock_kwargs.mock_calls == [call()] assert mock_logger.mock_calls == [ call.info("Connected to %s in region %s", @@ -107,6 +160,46 @@ def test_connect(self): bar='barval' ) ] + assert m_mrc.mock_calls == [call()] + assert cls.conn == mock_client.return_value + + def test_connect_with_retries(self): + mock_conn = Mock() + mock_cc = Mock() + type(mock_cc).region_name = 'myregion' + type(mock_conn)._client_config = mock_cc + + cls = ConnectableTester() + cls.api_name = 'myapi' + kwargs = {'foo': 'fooval', 'bar': 'barval'} + mock_conf = Mock() + + with patch('%s._boto3_connection_kwargs' % pb, + new_callable=PropertyMock, create=True) as mock_kwargs: + mock_kwargs.return_value = kwargs + with patch('%s.logger' % pbm) as mock_logger: + with patch('%s.boto3.client' % pbm) as mock_client: + with patch( + '%s._max_retries_config' % pb, new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = mock_conf + mock_client.return_value = mock_conn + cls.connect() + assert mock_kwargs.mock_calls == [call()] + assert mock_logger.mock_calls == [ + call.info("Connected to %s in region %s", + 'myapi', + 'myregion') + ] + assert mock_client.mock_calls == [ + call( + 'myapi', + foo='fooval', + bar='barval', + config=mock_conf + ) + ] + assert m_mrc.mock_calls == [call(), call()] assert cls.conn == mock_client.return_value def test_connect_again(self): @@ -125,11 +218,17 @@ def test_connect_again(self): mock_kwargs.return_value = kwargs with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.client' % pbm) as mock_client: - mock_client.return_value = mock_conn - cls.connect() + with patch( + '%s._max_retries_config' % pb, + new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = None + mock_client.return_value = mock_conn + cls.connect() assert mock_kwargs.mock_calls == [] assert mock_logger.mock_calls == [] assert mock_client.mock_calls == [] + assert m_mrc.mock_calls == [] assert cls.conn == mock_conn def test_connect_resource(self): @@ -151,8 +250,13 @@ def test_connect_resource(self): mock_kwargs.return_value = kwargs with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.resource' % pbm) as mock_resource: - mock_resource.return_value = mock_conn - cls.connect_resource() + with patch( + '%s._max_retries_config' % pb, + new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = None + mock_resource.return_value = mock_conn + cls.connect_resource() assert mock_kwargs.mock_calls == [call()] assert mock_logger.mock_calls == [ call.info("Connected to %s (resource) in region %s", @@ -166,6 +270,51 @@ def test_connect_resource(self): bar='barval' ) ] + assert m_mrc.mock_calls == [call()] + assert cls.resource_conn == mock_resource.return_value + + def test_connect_resource_with_max_retries(self): + mock_conn = Mock() + mock_meta = Mock() + mock_client = Mock() + mock_cc = Mock() + type(mock_cc).region_name = 'myregion' + type(mock_client)._client_config = mock_cc + type(mock_meta).client = mock_client + type(mock_conn).meta = mock_meta + + cls = ConnectableTester() + cls.api_name = 'myapi' + kwargs = {'foo': 'fooval', 'bar': 'barval'} + mock_conf = Mock() + + with patch('%s._boto3_connection_kwargs' % pb, + new_callable=PropertyMock, create=True) as mock_kwargs: + mock_kwargs.return_value = kwargs + with patch('%s.logger' % pbm) as mock_logger: + with patch('%s.boto3.resource' % pbm) as mock_resource: + with patch( + '%s._max_retries_config' % pb, + new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = mock_conf + mock_resource.return_value = mock_conn + cls.connect_resource() + assert mock_kwargs.mock_calls == [call()] + assert mock_logger.mock_calls == [ + call.info("Connected to %s (resource) in region %s", + 'myapi', + 'myregion') + ] + assert mock_resource.mock_calls == [ + call( + 'myapi', + foo='fooval', + bar='barval', + config=mock_conf + ) + ] + assert m_mrc.mock_calls == [call(), call()] assert cls.resource_conn == mock_resource.return_value def test_connect_resource_again(self): @@ -189,11 +338,17 @@ def test_connect_resource_again(self): mock_kwargs.return_value = kwargs with patch('%s.logger' % pbm) as mock_logger: with patch('%s.boto3.resource' % pbm) as mock_resource: - mock_resource.return_value = mock_conn - cls.connect_resource() + with patch( + '%s._max_retries_config' % pb, + new_callable=PropertyMock + ) as m_mrc: + m_mrc.return_value = None + mock_resource.return_value = mock_conn + cls.connect_resource() assert mock_kwargs.mock_calls == [] assert mock_logger.mock_calls == [] assert mock_resource.mock_calls == [] + assert m_mrc.mock_calls == [] assert cls.resource_conn == mock_conn diff --git a/awslimitchecker/tests/test_limit.py b/awslimitchecker/tests/test_limit.py index a41666ca..8707dd2f 100644 --- a/awslimitchecker/tests/test_limit.py +++ b/awslimitchecker/tests/test_limit.py @@ -41,7 +41,7 @@ import sys from awslimitchecker.limit import ( AwsLimit, AwsLimitUsage, SOURCE_DEFAULT, SOURCE_OVERRIDE, - SOURCE_TA, SOURCE_API + SOURCE_TA, SOURCE_API, SOURCE_QUOTAS ) from awslimitchecker.services.base import _AwsService @@ -56,13 +56,17 @@ from unittest.mock import patch, call, Mock -class TestAwsLimit(object): +class AwsLimitTester(object): def setup(self): self.mock_svc = Mock(spec_set=_AwsService) type(self.mock_svc).service_name = 'mysname' + type(self.mock_svc).quotas_service_code = 'qscode' - def test_init(self): + +class TestInit(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -84,8 +88,14 @@ def test_init(self): assert limit.def_critical_threshold == 11 assert limit._ta_service_name is None assert limit._ta_limit_name is None - - def test_init_ta_names(self): + assert limit._quotas_service_code is None + assert limit._quotas_name is None + assert limit._quotas_unit == 'None' + assert limit.quotas_limit is None + assert limit.quotas_unit_converter is None + + def test_ta_names(self): + m_foo = Mock() limit = AwsLimit( 'limitname', self.mock_svc, @@ -93,7 +103,11 @@ def test_init_ta_names(self): 7, 11, ta_service_name='foo', - ta_limit_name='bar' + ta_limit_name='bar', + quotas_service_code='baz', + quotas_name='blam', + quotas_unit='blarg', + quotas_unit_converter=m_foo ) assert limit.name == 'limitname' assert limit.service == self.mock_svc @@ -108,8 +122,13 @@ def test_init_ta_names(self): assert limit.def_critical_threshold == 11 assert limit._ta_service_name == 'foo' assert limit._ta_limit_name == 'bar' + assert limit._quotas_service_code == 'baz' + assert limit._quotas_name == 'blam' + assert limit._quotas_unit == 'blarg' + assert limit.quotas_limit is None + assert limit.quotas_unit_converter == m_foo - def test_init_valueerror(self): + def test_valueerror(self): with pytest.raises(ValueError) as excinfo: AwsLimit( 'limitname', @@ -125,7 +144,7 @@ def test_init_valueerror(self): assert msg == "critical threshold must be greater " \ "than warning threshold" - def test_init_type(self): + def test_type(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -145,7 +164,10 @@ def test_init_type(self): assert limit.def_warning_threshold == 6 assert limit.def_critical_threshold == 12 - def test_set_limit_override(self): + +class TestSetLimitOverride(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -158,7 +180,7 @@ def test_set_limit_override(self): assert limit.default_limit == 3 assert limit.override_ta is True - def test_set_limit_override_ta_False(self): + def test_ta_False(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -171,7 +193,10 @@ def test_set_limit_override_ta_False(self): assert limit.default_limit == 3 assert limit.override_ta is False - def test_set_ta_limit(self): + +class TestSetTaLimit(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -197,7 +222,10 @@ def test_set_ta_unlimited(self): assert limit.ta_limit is None assert limit.ta_unlimited is True - def test_set_api_limit(self): + +class TestSetApiLimit(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -209,7 +237,25 @@ def test_set_api_limit(self): limit._set_api_limit(10) assert limit.api_limit == 10 - def test_add_current_usage(self): + +class TestSetQuotasLimit(AwsLimitTester): + + def test_simple(self): + limit = AwsLimit( + 'limitname', + self.mock_svc, + 3, + 1, + 2 + ) + assert limit.quotas_limit is None + limit._set_quotas_limit(10.1) + assert limit.quotas_limit == 10.1 + + +class TestAddCurrentUsage(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -225,7 +271,10 @@ def test_add_current_usage(self): assert len(limit.get_current_usage()) == 2 assert limit._current_usage[1].get_value() == 4 - def test_get_current_usage(self): + +class TestGetCurrentUsage(AwsLimitTester): + + def test_simple(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -236,7 +285,7 @@ def test_get_current_usage(self): limit._current_usage = 2 assert limit.get_current_usage() == 2 - def test_get_current_usage_str_none(self): + def test_str_none(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -246,7 +295,7 @@ def test_get_current_usage_str_none(self): ) assert limit.get_current_usage_str() == '' - def test_get_current_usage_str(self): + def test_str(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -257,7 +306,7 @@ def test_get_current_usage_str(self): limit._add_current_usage(4) assert limit.get_current_usage_str() == '4' - def test_get_current_usage_str_id(self): + def test_str_id(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -268,7 +317,7 @@ def test_get_current_usage_str_id(self): limit._add_current_usage(4, resource_id='foobar') assert limit.get_current_usage_str() == 'foobar=4' - def test_get_current_usage_str_multi(self): + def test_str_multi(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -281,7 +330,7 @@ def test_get_current_usage_str_multi(self): limit._add_current_usage(2) assert limit.get_current_usage_str() == 'max: 4 (2, 3, 4)' - def test_get_current_usage_str_multi_id(self): + def test_str_multi_id(self): limit = AwsLimit( 'limitname', self.mock_svc, @@ -295,107 +344,167 @@ def test_get_current_usage_str_multi_id(self): assert limit.get_current_usage_str() == 'max: foo4bar=4 (foo2bar=2, ' \ 'foo3bar=3, foo4bar=4)' - def test_get_limit_default(self): + +class TestGetLimit(AwsLimitTester): + + def test_default(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) assert limit.get_limit() == 3 - def test_get_limit_override(self): + def test_override(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55) assert limit.get_limit() == 55 - def test_get_limit_ta(self): + def test_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55, override_ta=False) limit._set_ta_limit(40) assert limit.get_limit() == 40 - def test_get_limit_ta_unlimited(self): + def test_ta_unlimited(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55, override_ta=False) limit._set_ta_unlimited() assert limit.get_limit() is None - def test_get_limit_api(self): + def test_api(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_api_limit(40) assert limit.get_limit() == 40 - def test_get_limit_api_ta(self): + def test_api_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_ta_limit(40) limit._set_api_limit(11) assert limit.get_limit() == 11 - def test_get_limit_api_override(self): + def test_api_override(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55) limit._set_api_limit(40) assert limit.get_limit() == 55 - def test_get_limit_api_override_ta(self): + def test_api_override_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55) limit._set_ta_limit(40) limit._set_api_limit(11) assert limit.get_limit() == 55 - def test_get_limit_source_default(self): + def test_quotas(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(12) + assert limit.get_limit() == 12 + + def test_quotas_api_ta_override(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit.set_limit_override(55) + limit._set_ta_limit(40) + limit._set_api_limit(11) + limit._set_quotas_limit(12) + assert limit.get_limit() == 55 + + +class TestGetLimitSource(AwsLimitTester): + + def test_default(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) assert limit.get_limit_source() == SOURCE_DEFAULT - def test_get_limit_source_override(self): + def test_override(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55) assert limit.get_limit_source() == SOURCE_OVERRIDE - def test_get_limit_source_override_no_override_ta(self): + def test_override_no_override_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55, override_ta=False) limit._set_ta_limit(40) assert limit.get_limit_source() == SOURCE_TA - def test_get_limit_source_override_with_ta(self): + def test_override_with_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit.set_limit_override(55) limit._set_ta_limit(40) assert limit.get_limit_source() == SOURCE_OVERRIDE - def test_get_limit_source_ta(self): + def test_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_ta_limit(40) assert limit.get_limit_source() == SOURCE_TA - def test_get_limit_source_ta_unlimited(self): + def test_ta_unlimited(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_ta_unlimited() assert limit.get_limit_source() == SOURCE_TA - def test_get_limit_source_api(self): + def test_api(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_api_limit(40) assert limit.get_limit_source() == SOURCE_API - def test_get_limit_source_api_override(self): + def test_api_override(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_api_limit(40) limit.set_limit_override(55) assert limit.get_limit_source() == SOURCE_OVERRIDE - def test_get_limit_source_api_ta(self): + def test_api_ta(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_api_limit(40) + limit._set_ta_limit(41) + assert limit.get_limit_source() == SOURCE_API + + def test_api_override_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) limit._set_api_limit(40) limit._set_ta_limit(41) + limit.set_limit_override(55) + assert limit.get_limit_source() == SOURCE_OVERRIDE + + def test_quotas(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(40) + assert limit.get_limit_source() == SOURCE_QUOTAS + + def test_quotas_api(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_api_limit(40) + limit._set_quotas_limit(50) assert limit.get_limit_source() == SOURCE_API - def test_get_limit_source_api_override_ta(self): + def test_quotas_override(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(40) + limit.set_limit_override(55) + assert limit.get_limit_source() == SOURCE_OVERRIDE + + def test_quotas_ta(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(40) + limit._set_ta_limit(41) + assert limit.get_limit_source() == SOURCE_QUOTAS + + def test_quotas_override_ta(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(40) + limit._set_ta_limit(41) + limit.set_limit_override(55) + assert limit.get_limit_source() == SOURCE_OVERRIDE + + def test_quotas_api_override_ta(self): + limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) + limit._set_quotas_limit(10.2) limit._set_api_limit(40) limit._set_ta_limit(41) limit.set_limit_override(55) assert limit.get_limit_source() == SOURCE_OVERRIDE - def test_check_thresholds_pct(self): + +class TestCheckThresholds(AwsLimitTester): + + def test_pct(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 3, resource_id='foo3bar') @@ -414,7 +523,7 @@ def test_check_thresholds_pct(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_ta_unlimited(self): + def test_ta_unlimited(self): limit = AwsLimit('limitname', self.mock_svc, 3, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 3, resource_id='foo3bar') @@ -434,7 +543,7 @@ def test_check_thresholds_ta_unlimited(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_pct_warn(self): + def test_pct_warn(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 50, resource_id='foo3bar') @@ -453,7 +562,7 @@ def test_check_thresholds_pct_warn(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_int_warn(self): + def test_int_warn(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 1, resource_id='foo3bar') @@ -472,7 +581,7 @@ def test_check_thresholds_int_warn(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_int_warn_crit(self): + def test_int_warn_crit(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 1, resource_id='foo3bar') @@ -491,7 +600,7 @@ def test_check_thresholds_int_warn_crit(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_pct_crit(self): + def test_pct_crit(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 4, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 3, resource_id='foo3bar') @@ -510,7 +619,7 @@ def test_check_thresholds_pct_crit(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_int_crit(self): + def test_int_crit(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 9, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 3, resource_id='foo3bar') @@ -529,7 +638,7 @@ def test_check_thresholds_int_crit(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_check_thresholds_pct_warn_crit(self): + def test_pct_warn_crit(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) u1 = AwsLimitUsage(limit, 50, resource_id='foo4bar') u2 = AwsLimitUsage(limit, 3, resource_id='foo3bar') @@ -548,19 +657,28 @@ def test_check_thresholds_pct_warn_crit(self): assert mock_get_thresh.mock_calls == [call()] assert mock_get_limit.mock_calls == [call(), call(), call()] - def test_get_warnings(self): + +class TestGetWarnings(AwsLimitTester): + + def test_simple(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) m = Mock() limit._warnings = m assert limit.get_warnings() == m - def test_get_criticals(self): + +class TestGetCriticals(AwsLimitTester): + + def test_simple(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) m = Mock() limit._criticals = m assert limit.get_criticals() == m - def test_get_thresholds(self): + +class TestGetThresholds(AwsLimitTester): + + def test_simple(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) assert limit._get_thresholds() == ( None, @@ -569,7 +687,7 @@ def test_get_thresholds(self): 2 ) - def test_get_thresholds_overridden(self): + def test_overridden(self): limit = AwsLimit('limitname', self.mock_svc, 100, 88, 99) limit.warn_percent = 1 limit.warn_count = 2 @@ -582,6 +700,9 @@ def test_get_thresholds_overridden(self): 3 ) + +class TestSetThresholdOverride(AwsLimitTester): + def test_set_threshold_override(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) limit.set_threshold_override( @@ -595,27 +716,66 @@ def test_set_threshold_override(self): assert limit.crit_percent == 3 assert limit.crit_count == 4 - def test_ta_service_name_default(self): + +class TestTaServiceName(AwsLimitTester): + + def test_default(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) assert limit.ta_service_name == 'mysname' - def test_ta_service_name_overridden(self): + def test_overridden(self): limit = AwsLimit( 'limitname', self.mock_svc, 100, 1, 2, ta_service_name='foo' ) assert limit.ta_service_name == 'foo' - def test_ta_limit_name_default(self): + +class TestTaLimitName(AwsLimitTester): + + def test_default(self): limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) assert limit.ta_limit_name == 'limitname' - def test_ta_limit_name_overridden(self): + def test_overridden(self): limit = AwsLimit( 'limitname', self.mock_svc, 100, 1, 2, ta_limit_name='foo' ) assert limit.ta_limit_name == 'foo' +class TestQuotasServiceCode(AwsLimitTester): + + def test_default(self): + limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) + assert limit.quotas_service_code == 'qscode' + + def test_overridden(self): + limit = AwsLimit( + 'limitname', self.mock_svc, 100, 1, 2, quotas_service_code='qsc' + ) + assert limit.quotas_service_code == 'qsc' + + +class TestQuotaName(AwsLimitTester): + + def test_default(self): + limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) + assert limit.quota_name == 'limitname' + + def test_overridden(self): + limit = AwsLimit( + 'limitname', self.mock_svc, 100, 1, 2, quotas_name='qn' + ) + assert limit.quota_name == 'qn' + + +class TestQuotasUnit(AwsLimitTester): + + def test_default(self): + limit = AwsLimit('limitname', self.mock_svc, 100, 1, 2) + assert limit.quotas_unit == 'None' + + class TestAwsLimitUsage(object): def test_init(self): diff --git a/awslimitchecker/tests/test_quotas.py b/awslimitchecker/tests/test_quotas.py new file mode 100644 index 00000000..5f015c27 --- /dev/null +++ b/awslimitchecker/tests/test_quotas.py @@ -0,0 +1,240 @@ +""" +awslimitchecker/tests/test_quotas.py + +The latest version of this package is available at: + + +############################################################################## +Copyright 2015-2019 Jason Antman + + This file is part of awslimitchecker, also known as awslimitchecker. + + awslimitchecker is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + awslimitchecker is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with awslimitchecker. If not, see . + +The Copyright and Authors attributions contained herein may not be removed or +otherwise altered, except to add the Author attribution of a contributor to +this work. (Additional Terms pursuant to Section 7b of the AGPL v3) +############################################################################## +While not legally required, I sincerely request that anyone who finds +bugs please submit them at or +to me via email, and that you send any contributions or improvements +either as a pull request on GitHub, or to me via email. +############################################################################## + +AUTHORS: +Jason Antman +############################################################################## +""" + +import sys +from botocore.exceptions import ClientError +import pytest + +from awslimitchecker.quotas import ServiceQuotasClient +from awslimitchecker.tests.support import quotas_response + +# https://code.google.com/p/mock/issues/detail?id=249 +# py>=3.4 should use unittest.mock not the mock package on pypi +if ( + sys.version_info[0] < 3 or + sys.version_info[0] == 3 and sys.version_info[1] < 4 +): + from mock import patch, call, Mock +else: + from unittest.mock import patch, call, Mock + +pbm = 'awslimitchecker.quotas' +pb = '%s.ServiceQuotasClient' % pbm + + +class TestConstructor(object): + + def test_init(self): + cls = ServiceQuotasClient({'foo': 'bar'}) + assert cls._boto3_connection_kwargs == {'foo': 'bar'} + assert cls._cache == {} + assert cls.conn is None + + +class TestQuotasForService(object): + + def setup(self): + self.cls = ServiceQuotasClient({'foo': 'bar'}) + + def test_not_cached(self): + resp, expected = quotas_response() + mock_paginator = Mock() + mock_paginator.paginate.return_value = resp + mock_conn = Mock() + mock_conn.get_paginator.return_value = mock_paginator + + def se_connect(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_connect + res = self.cls.quotas_for_service('scode') + assert res == expected + assert self.cls._cache == {'scode': expected} + assert m_connect.mock_calls == [call(self.cls)] + assert mock_conn.mock_calls == [ + call.get_paginator('list_service_quotas'), + call.get_paginator().paginate(ServiceCode='scode') + ] + + def test_cached(self): + resp, expected = quotas_response() + mock_paginator = Mock() + mock_paginator.paginate.return_value = resp + mock_conn = Mock() + mock_conn.get_paginator.return_value = mock_paginator + + def se_connect(cls): + cls.conn = mock_conn + + self.cls._cache = {'scode': expected} + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_connect + res = self.cls.quotas_for_service('scode') + assert res == expected + assert self.cls._cache == {'scode': expected} + assert m_connect.mock_calls == [] + assert mock_conn.mock_calls == [] + + def test_no_such_resource(self): + mock_paginator = Mock() + mock_paginator.paginate.side_effect = ClientError( + { + 'Error': { + 'Code': 'NoSuchResourceException', + 'Message': 'The request failed because the specified ' + 'service does not exist.' + } + }, + 'ListServiceQuotas' + ) + mock_conn = Mock() + mock_conn.get_paginator.return_value = mock_paginator + + def se_connect(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_connect + res = self.cls.quotas_for_service('scode') + assert res == {} + assert self.cls._cache == {'scode': {}} + assert m_connect.mock_calls == [call(self.cls)] + assert mock_conn.mock_calls == [ + call.get_paginator('list_service_quotas'), + call.get_paginator().paginate(ServiceCode='scode') + ] + + def test_other_exception(self): + mock_paginator = Mock() + mock_paginator.paginate.side_effect = ClientError( + { + 'Error': { + 'Code': 'SomeOtherError', + 'Message': 'My message.' + } + }, + 'ListServiceQuotas' + ) + mock_conn = Mock() + mock_conn.get_paginator.return_value = mock_paginator + + def se_connect(cls): + cls.conn = mock_conn + + with patch('%s.connect' % pb, autospec=True) as m_connect: + m_connect.side_effect = se_connect + with pytest.raises(ClientError): + self.cls.quotas_for_service('scode') + assert self.cls._cache == {'scode': {}} + assert m_connect.mock_calls == [call(self.cls)] + assert mock_conn.mock_calls == [ + call.get_paginator('list_service_quotas'), + call.get_paginator().paginate(ServiceCode='scode') + ] + + +class TestGetQuotaValue(object): + + def setup(self): + self.cls = ServiceQuotasClient({'foo': 'bar'}) + + def test_happy_path(self): + self.cls._cache = { + 'scode': { + 'qname': { + 'QuotaName': 'qname', + 'QuotaCode': 'qcode', + 'Value': 12.3, + 'Unit': 'None' + } + } + } + res = self.cls.get_quota_value('scode', 'QName') + assert res == 12.3 + + def test_no_quota(self): + self.cls._cache = { + 'scode': { + 'qname': { + 'QuotaName': 'qname', + 'QuotaCode': 'qcode', + 'Value': 12.3, + 'Unit': 'None' + } + } + } + res = self.cls.get_quota_value('scode', 'OtherName') + assert res is None + + def test_units(self): + self.cls._cache = { + 'scode': { + 'qname': { + 'QuotaName': 'qname', + 'QuotaCode': 'qcode', + 'Value': 12.3, + 'Unit': 'Foo' + } + } + } + res = self.cls.get_quota_value('scode', 'QName') + assert res is None + + def test_units_converter(self): + m_conv = Mock() + m_conv.return_value = 1230.0 + self.cls._cache = { + 'scode': { + 'qname': { + 'QuotaName': 'qname', + 'QuotaCode': 'qcode', + 'Value': 12.3, + 'Unit': 'Foo' + } + } + } + res = self.cls.get_quota_value( + 'scode', 'QName', converter=m_conv + ) + assert res == 1230.0 + assert m_conv.mock_calls == [ + call(12.3, 'Foo', 'None') + ] diff --git a/awslimitchecker/tests/test_runner.py b/awslimitchecker/tests/test_runner.py index d0615be1..ac234c63 100644 --- a/awslimitchecker/tests/test_runner.py +++ b/awslimitchecker/tests/test_runner.py @@ -125,6 +125,9 @@ def test_simple(self): assert res.list_alert_providers is False assert res.alert_provider is None assert res.alert_config == {} + assert res.role_partition == 'aws' + assert res.ta_api_region == 'us-east-1' + assert res.skip_quotas is False def test_parser(self): argv = ['-V'] @@ -229,9 +232,24 @@ def test_parser(self): type=str, default=None, help='AWS region name to connect to; required ' 'for STS'), + call().add_argument('--role-partition', action='store', type=str, + default='aws', + help='AWS partition name to use for ' + 'account_role when connecting via STS; ' + 'see documentation for more information' + ' (default: "aws")'), + call().add_argument('--ta-api-region', action='store', type=str, + default='us-east-1', + help='Region to use for Trusted Advisor / ' + 'Support API (default: us-east-1)'), call().add_argument('--skip-ta', action='store_true', default=False, help='do not attempt to pull *any* information ' 'on limits from Trusted Advisor'), + call().add_argument('--skip-quotas', action='store_true', + default=False, + help='Do not attempt to connect to Service ' + 'Quotas service or use its data for ' + 'current limits'), call().add_mutually_exclusive_group(), call().add_mutually_exclusive_group().add_argument( '--ta-refresh-wait', action='store_true', default=False, @@ -327,6 +345,12 @@ def test_ta_refresh_trigger(self): assert isinstance(res, argparse.Namespace) assert res.ta_refresh_mode == 'trigger' + def test_skip_quotas(self): + argv = ['--skip-quotas'] + res = self.cls.parse_args(argv) + assert isinstance(res, argparse.Namespace) + assert res.skip_quotas is True + def test_ta_refresh_older(self): argv = ['--ta-refresh-older=123'] res = self.cls.parse_args(argv) @@ -396,6 +420,15 @@ def test_alert_provider(self): assert res.alert_provider == 'ClassName' assert res.alert_config == {'foo': 'bar', 'baz': 'blam'} + def test_role_partition_ta_api_region(self): + argv = [ + '--role-partition=foo', + '--ta-api-region=bar' + ] + res = self.cls.parse_args(argv) + assert res.role_partition == 'foo' + assert res.ta_api_region == 'bar' + class TestListServices(RunnerTester): @@ -499,7 +532,8 @@ def test_simple(self, capsys): 'SvcBar/barlimit1': '1', 'SvcFoo/foo limit3': '10 (TA)', 'SvcFoo/zzz limit4': '34 (API)', - 'SvcFoo/limit with usage maximums/res_id': '10 (API)' + 'SvcFoo/limit with usage maximums/res_id': '10 (API)', + 'SvcFoo/zzz limit5': '60.0 (Quotas)' }) ] @@ -522,7 +556,8 @@ def test_one_service(self, capsys): call({ 'SvcFoo/foo limit3': '10 (TA)', 'SvcFoo/zzz limit4': '34 (API)', - 'SvcFoo/limit with usage maximums/res_id': '10 (API)' + 'SvcFoo/limit with usage maximums/res_id': '10 (API)', + 'SvcFoo/zzz limit5': '60.0 (Quotas)' }) ] @@ -1136,7 +1171,10 @@ def test_version(self, capsys): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ), call().get_project_url(), call().get_version() @@ -1165,8 +1203,8 @@ def test_iam_policy(self): call(self.cls) ] - def test_list_defaults(self): - argv = ['awslimitchecker', '--list-defaults'] + def test_list_defaults_skip_quotas(self): + argv = ['awslimitchecker', '--list-defaults', '--skip-quotas'] with patch.object(sys, 'argv', argv): with patch('%s.Runner.list_defaults' % pb, autospec=True) as mock_list: @@ -1204,7 +1242,46 @@ def test_skip_service_none(self): external_id=None, mfa_serial_number=None, mfa_token=None, profile_name=None, region=None, ta_refresh_mode=None, ta_refresh_timeout=None, warning_threshold=80, - check_version=True) + check_version=True, role_partition='aws', + ta_api_region='us-east-1', skip_quotas=False) + ] + + def test_role_partition(self): + argv = ['awslimitchecker', '--role-partition=foo'] + with patch.object(sys, 'argv', argv): + with patch('%s.Runner.check_thresholds' % pb, + autospec=True) as mock_check: + mock_check.return_value = 2, {'Foo': {'Bar': Mock()}}, 'foo' + with patch('%s.AwsLimitChecker' % pb, autospec=True) as mock_c: + with pytest.raises(SystemExit) as excinfo: + self.cls.console_entry_point() + assert excinfo.value.code == 2 + assert mock_c.mock_calls == [ + call(account_id=None, account_role=None, critical_threshold=99, + external_id=None, mfa_serial_number=None, mfa_token=None, + profile_name=None, region=None, ta_refresh_mode=None, + ta_refresh_timeout=None, warning_threshold=80, + check_version=True, role_partition='foo', + ta_api_region='us-east-1', skip_quotas=False) + ] + + def test_ta_api_region_skip_quotas(self): + argv = ['awslimitchecker', '--ta-api-region=foo', '--skip-quotas'] + with patch.object(sys, 'argv', argv): + with patch('%s.Runner.check_thresholds' % pb, + autospec=True) as mock_check: + mock_check.return_value = 2, {'Foo': {'Bar': Mock()}}, 'foo' + with patch('%s.AwsLimitChecker' % pb, autospec=True) as mock_c: + with pytest.raises(SystemExit) as excinfo: + self.cls.console_entry_point() + assert excinfo.value.code == 2 + assert mock_c.mock_calls == [ + call(account_id=None, account_role=None, critical_threshold=99, + external_id=None, mfa_serial_number=None, mfa_token=None, + profile_name=None, region=None, ta_refresh_mode=None, + ta_refresh_timeout=None, warning_threshold=80, + check_version=True, role_partition='aws', + ta_api_region='foo', skip_quotas=True) ] def test_skip_service(self): @@ -1222,7 +1299,8 @@ def test_skip_service(self): external_id=None, mfa_serial_number=None, mfa_token=None, profile_name=None, region=None, ta_refresh_mode=None, ta_refresh_timeout=None, warning_threshold=80, - check_version=True), + check_version=True, role_partition='aws', + ta_api_region='us-east-1', skip_quotas=False), call().remove_services(['foo']) ] @@ -1245,7 +1323,8 @@ def test_skip_service_multi(self): external_id=None, mfa_serial_number=None, mfa_token=None, profile_name=None, region=None, ta_refresh_mode=None, ta_refresh_timeout=None, warning_threshold=80, - check_version=True), + check_version=True, role_partition='aws', + ta_api_region='us-east-1', skip_quotas=False), call().remove_services(['foo', 'bar']) ] @@ -1267,7 +1346,8 @@ def test_skip_check(self): external_id=None, mfa_serial_number=None, mfa_token=None, profile_name=None, region=None, ta_refresh_mode=None, ta_refresh_timeout=None, warning_threshold=80, - check_version=True), + check_version=True, role_partition='aws', + ta_api_region='us-east-1', skip_quotas=False), ] assert self.cls.skip_check == [ 'EC2/Max launch specifications per spot fleet', @@ -1292,7 +1372,8 @@ def test_skip_check_multi(self): external_id=None, mfa_serial_number=None, mfa_token=None, profile_name=None, region=None, ta_refresh_mode=None, ta_refresh_timeout=None, warning_threshold=80, - check_version=True), + check_version=True, role_partition='aws', + ta_api_region='us-east-1', skip_quotas=False), ] assert self.cls.skip_check == [ 'EC2/Max launch specifications per spot fleet', @@ -1442,7 +1523,10 @@ def test_no_service_name_region(self, capsys): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] assert self.cls.service_name is None @@ -1481,7 +1565,10 @@ def test_no_service_name_sts(self, capsys): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] assert self.cls.service_name is None @@ -1523,7 +1610,10 @@ def test_no_service_name_sts_external_id(self, capsys): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=False + check_version=False, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] assert self.cls.service_name is None @@ -1581,7 +1671,10 @@ def test_warning(self): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] @@ -1608,7 +1701,10 @@ def test_warning_profile_name(self): profile_name='myprof', ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] @@ -1635,7 +1731,10 @@ def test_critical(self): profile_name=None, ta_refresh_mode=None, ta_refresh_timeout=None, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] @@ -1663,7 +1762,10 @@ def test_critical_ta_refresh(self): profile_name=None, ta_refresh_mode=456, ta_refresh_timeout=123, - check_version=True + check_version=True, + role_partition='aws', + ta_api_region='us-east-1', + skip_quotas=False ) ] diff --git a/awslimitchecker/tests/test_trustedadvisor.py b/awslimitchecker/tests/test_trustedadvisor.py index e3dea806..61f81bdd 100644 --- a/awslimitchecker/tests/test_trustedadvisor.py +++ b/awslimitchecker/tests/test_trustedadvisor.py @@ -116,6 +116,33 @@ def test_boto_kwargs(self): assert cls.refresh_mode == 123 assert cls.refresh_timeout == 456 + def test_boto_kwargs_ta_api_region(self): + mock_svc = Mock(spec_set=_AwsService) + mock_svc.get_limits.return_value = {} + boto_args = dict(region_name='myregion', + aws_access_key_id='myaccesskey', + aws_secret_access_key='mysecretkey', + aws_session_token='mytoken') + + cls = TrustedAdvisor( + {'foo': mock_svc}, + boto_args, + ta_refresh_mode=123, + ta_refresh_timeout=456, + ta_api_region='myRegion' + ) + assert cls.conn is None + cls_boto_args = cls._boto3_connection_kwargs + assert cls_boto_args.get('region_name') == 'myRegion' + assert cls_boto_args.get('aws_access_key_id') == 'myaccesskey' + assert cls_boto_args.get('aws_secret_access_key') == 'mysecretkey' + assert cls_boto_args.get('aws_session_token') == 'mytoken' + assert cls.ta_region == 'myregion' + assert cls.all_services == {'foo': mock_svc} + assert cls.limits_updated is False + assert cls.refresh_mode == 123 + assert cls.refresh_timeout == 456 + class TestUpdateLimits(object): @@ -325,10 +352,10 @@ def setup(self): def test_none(self): tmp = self.mock_conn.describe_trusted_advisor_check_result with patch('%s._get_limit_check_id' % pb, autospec=True) as mock_id: - mock_id.return_value = None + mock_id.return_value = (None, None) res = self.cls._poll() assert tmp.mock_calls == [] - assert res is None + assert res == {} def test_basic(self): poll_return_val = { diff --git a/awslimitchecker/trustedadvisor.py b/awslimitchecker/trustedadvisor.py index 296dcd84..8dc4d7ab 100644 --- a/awslimitchecker/trustedadvisor.py +++ b/awslimitchecker/trustedadvisor.py @@ -59,7 +59,8 @@ class TrustedAdvisor(Connectable): api_name = 'support' def __init__(self, all_services, boto_connection_kwargs, - ta_refresh_mode=None, ta_refresh_timeout=None): + ta_refresh_mode=None, ta_refresh_timeout=None, + ta_api_region='us-east-1'): """ Class to contain all TrustedAdvisor-related logic. @@ -110,13 +111,16 @@ def __init__(self, all_services, boto_connection_kwargs, parameter is not None, only wait up to this number of seconds for the refresh to finish before continuing on anyway. :type ta_refresh_timeout: :py:class:`int` or :py:data:`None` + :param ta_api_region: The AWS region used for calls to the + TrustedAdvisor API. This is always us-east-1 for + non GovCloud accounts. + :type ta_api_region: str """ self.conn = None self.have_ta = True self.ta_region = boto_connection_kwargs.get('region_name') - # All Support/TA API connections are to us-east-1 only ta_kwargs = deepcopy(boto_connection_kwargs) - ta_kwargs['region_name'] = 'us-east-1' + ta_kwargs['region_name'] = ta_api_region self._boto3_connection_kwargs = ta_kwargs self.refresh_mode = ta_refresh_mode self.refresh_timeout = ta_refresh_timeout @@ -164,10 +168,10 @@ def _poll(self): if not self.have_ta: logger.info('TrustedAdvisor.have_ta is False; not polling TA') return {} - if tmp is None: + if tmp[0] is None: logger.critical("Unable to find 'Service Limits' Trusted Advisor " "check; not using Trusted Advisor data.") - return + return {} check_id, metadata = tmp checks = self._get_refreshed_check_result(check_id) region = self.ta_region or self.conn._client_config.region_name diff --git a/awslimitchecker/version.py b/awslimitchecker/version.py index 853b1ed7..512fe5d5 100644 --- a/awslimitchecker/version.py +++ b/awslimitchecker/version.py @@ -47,7 +47,7 @@ except ImportError: logger.error("Unable to import versionfinder", exc_info=True) -_VERSION_TUP = (7, 1, 0) +_VERSION_TUP = (8, 0, 0) _VERSION = '.'.join([str(x) for x in _VERSION_TUP]) _PROJECT_URL = 'https://github.com/jantman/awslimitchecker' diff --git a/dev/missing_instance_types.py b/dev/missing_instance_types.py index 4f91a681..19980e57 100755 --- a/dev/missing_instance_types.py +++ b/dev/missing_instance_types.py @@ -35,3 +35,11 @@ )) if len(missing) > 0: print('Missing types: %s' % missing) +extra = [] +for itype in sorted(alctypes): + if itype not in uniq: + print('EXTRA INSTANCE TYPE: %s' % itype) + extra.append(itype) +print('awslimitchecker currently has %d extra instance types' % len(extra)) +if len(extra) > 0: + print('Extra types: %s' % extra) diff --git a/docs/build_generated_docs.py b/docs/build_generated_docs.py index 88168d64..c4f03947 100644 --- a/docs/build_generated_docs.py +++ b/docs/build_generated_docs.py @@ -49,6 +49,9 @@ my_dir = os.path.dirname(os.path.abspath(__file__)) os.environ['PYTHONPATH'] = os.path.join(my_dir, '..') sys.path.insert(0, os.path.join(my_dir, '..')) +# always run in us-east-1, because some Quotas Service quotas only exist there +os.environ['AWS_REGION'] = 'us-east-1' +os.environ['AWS_DEFAULT_REGION'] = 'us-east-1' from awslimitchecker.checker import AwsLimitChecker from awslimitchecker.metrics import MetricsProvider @@ -108,6 +111,104 @@ def build_iam_policy(checker): fh.write(doc) +def format_limits_for_service(limits): + limit_info = '' + # build a dict of the limits + slimits = {} + # track the maximum string lengths + max_name = 0 + max_default_limit = 0 + for limit in limits.values(): + slimits[limit.name] = limit + # update max string length for table formatting + if len(limit.name) > max_name: + max_name = len(limit.name) + if len(str(limit.default_limit)) > max_default_limit: + max_default_limit = len(str(limit.default_limit)) + # create the format string + sformat = '{name: <' + str(max_name) + '} ' \ + '{ta: <15} {quotas: <8} ' \ + '{api: <7} ' \ + '{limit: <' + str( + max_default_limit) + '}\n' + # separator lines + sep = ('=' * max_name) + ' =============== ======== ======= ' + \ + ('=' * max_default_limit) + "\n" + # header + limit_info += sep + limit_info += sformat.format( + name='Limit', limit='Default', api='API', ta='Trusted Advisor', + quotas='Quotas' + ) + limit_info += sep + # limit lines + for lname, limit in sorted(slimits.items()): + limit_info += sformat.format( + name=lname, limit=str(limit.default_limit), + ta='|check|' if limit.ta_limit is not None else '', + api='|check|' if ( + limit.api_limit is not None or limit.has_resource_limits() + ) else '', + quotas='|check|' if limit.quotas_limit is not None else '' + ) + # footer + limit_info += sep + limit_info += "\n" + return limit_info + + +def limits_for_ec2(): + limit_info = '.. _limits.EC2:\n\n' + limit_info += "EC2\n---\n\n" + limit_info += dedent(""" + As of October 2019, the "standard" EC2 regions use the new + `vCPU-based limits `__, while the China (``cn-``) and GovCloud (``us-gov-``) + regions still use the old per-instance-type limits. Please see the sections + for either :ref:`limits.ec2-standard` or :ref:`limits.ec2-nonvcpu` for + details. + + """) + limit_info += '.. _limits.ec2-standard:\n\n' + limit_info += "EC2 - Standard Regions\n" + limit_info += "----------------------\n" + limit_info += "\n" + dedent(""" + **Note on On-Demand vs Reserved Instances:** The EC2 limits for + "Running On-Demand" EC2 Instances apply only to On-Demand instances, + not Reserved Instances. If you list all EC2 instances that are + running in the Console or API, you'll get back instances of all types + (On-Demand, Reserved, etc.). The value that awslimitchecker reports + for Running On-Demand Instances current usage will *not* match the + number of instances you see in the Console or API. + + **Important:** The limits for **Running On-Demand Instances** are now + measured in vCPU count per instance family, not instance count per instance + type. + """) + "\n" + limit_info += "\n" + limit_info += format_limits_for_service( + AwsLimitChecker(region='us-east-1').get_limits()['EC2'] + ) + limit_info += '.. _limits.ec2-nonvcpu:\n\n' + limit_info += "EC2 - China and GovCloud\n" + limit_info += "------------------------\n" + limit_info += "\n" + dedent(""" + **Note on On-Demand vs Reserved Instances:** The EC2 limits for + "Running On-Demand" EC2 Instances apply only to On-Demand instances, + not Reserved Instances. If you list all EC2 instances that are + running in the Console or API, you'll get back instances of all types + (On-Demand, Reserved, etc.). The value that awslimitchecker reports + for Running On-Demand Instances current usage will *not* match the + number of instances you see in the Console or API. + """) + "\n" + limit_info += "\n" + fname = os.path.join(my_dir, 'source', 'ec2_nonvcpu_limits.txt') + with open(fname, 'r') as fh: + limit_info += fh.read() + limit_info += "\n\n" + return limit_info + + def build_limits(checker): logger.info("Beginning build of limits.rst") logger.info("Getting Limits") @@ -115,19 +216,12 @@ def build_limits(checker): limits = checker.get_limits() # this is a bit of a pain, because we need to know string lengths to build the table for svc_name in sorted(limits): + if svc_name == 'EC2': + limit_info += limits_for_ec2() + continue limit_info += '.. _limits.%s:\n\n' % svc_name limit_info += svc_name + "\n" limit_info += ('-' * (len(svc_name)+1)) + "\n" - if svc_name == 'EC2': - limit_info += "\n" + dedent(""" - **Note on On-Demand vs Reserved Instances:** The EC2 limits for - "Running On-Demand" EC2 Instances apply only to On-Demand instances, - not Reserved Instances. If you list all EC2 instances that are - running in the Console or API, you'll get back instances of all types - (On-Demand, Reserved, etc.). The value that awslimitchecker reports - for Running On-Demand Instances current usage will *not* match the - number of instances you see in the Console or API. - """) + "\n" if svc_name == 'Route53': limit_info += "\n" + dedent(""" **Note on Route53 Limits:** The Route53 limit values (maxima) are @@ -135,43 +229,7 @@ def build_limits(checker): zone. As such, each zone may have a different limit value. """) + "\n" limit_info += "\n" - # build a dict of the limits - slimits = {} - # track the maximum string lengths - max_name = 0 - max_default_limit = 0 - for limit in limits[svc_name].values(): - slimits[limit.name] = limit - # update max string length for table formatting - if len(limit.name) > max_name: - max_name = len(limit.name) - if len(str(limit.default_limit)) > max_default_limit: - max_default_limit = len(str(limit.default_limit)) - # create the format string - sformat = '{name: <' + str(max_name) + '} ' \ - '{ta: <15} {api: <7} ' \ - '{limit: <' + str(max_default_limit) + '}\n' - # separator lines - sep = ('=' * max_name) + ' =============== ======= ' + \ - ('=' * max_default_limit) + "\n" - # header - limit_info += sep - limit_info += sformat.format( - name='Limit', limit='Default', api='API', ta='Trusted Advisor' - ) - limit_info += sep - # limit lines - for lname, limit in sorted(slimits.iteritems()): - limit_info += sformat.format( - name=lname, limit=str(limit.default_limit), - ta='|check|' if limit.ta_limit is not None else '', - api='|check|' if ( - limit.api_limit is not None or limit.has_resource_limits() - ) else '' - ) - # footer - limit_info += sep - limit_info += "\n" + limit_info += format_limits_for_service(limits[svc_name]) doc = """ .. -- WARNING -- WARNING -- WARNING @@ -196,11 +254,17 @@ def build_limits(checker): last release. Note that not all accounts can access Trusted Advisor, or can access all limits known by Trusted Advisor. + **Limits with a** |check| **in the "Quotas" column can be retrieved from + the Service Quotas service**; this information is supposed to be accurate + and up-to-date, but is likely less accurate than the service's own API. + Limits retrieved from Service Quotas take precedence over Trusted Advisor + and default limits. + **Limits with a** |check| **in the "API" column can be retrieved directly from the corresponding Service API**; this information should be the most accurate and up-to-date, as it is retrieved directly from the service that evaluates and enforces limits. Limits retrieved via service API take precedence over - Trusted Advisor and default limits. + Trusted Advisor, Service Quotas, and default limits. {limit_info} @@ -225,6 +289,7 @@ def build_runner_examples(): 'list_limits': ['awslimitchecker', '-l'], 'list_defaults': ['awslimitchecker', '--list-defaults'], 'skip_ta': ['awslimitchecker', '-l', '--skip-ta'], + 'skip_quotas': ['awslimitchecker', '-l', '--skip-quotas'], 'show_usage': ['awslimitchecker', '-u'], 'list_services': ['awslimitchecker', '-s'], 'limit_overrides': [ @@ -247,9 +312,9 @@ def build_runner_examples(): cmd_str = ' '.join(command) logger.info("Running: %s", cmd_str) try: - output = subprocess.check_output(command) + output = subprocess.check_output(command).decode() except subprocess.CalledProcessError as e: - output = e.output + output = e.output.decode() results[name] = format_cmd_output(cmd_str, output, name) results['%s-output-only' % name] = format_cmd_output(None, output, name) results['metrics-providers'] = '' diff --git a/docs/source/awslimitchecker.alerts.base.rst b/docs/source/awslimitchecker.alerts.base.rst index ffb25be7..fd255f03 100644 --- a/docs/source/awslimitchecker.alerts.base.rst +++ b/docs/source/awslimitchecker.alerts.base.rst @@ -2,6 +2,6 @@ awslimitchecker.alerts.base module ================================== .. automodule:: awslimitchecker.alerts.base - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.alerts.dummy.rst b/docs/source/awslimitchecker.alerts.dummy.rst index b1c594e2..b54376c5 100644 --- a/docs/source/awslimitchecker.alerts.dummy.rst +++ b/docs/source/awslimitchecker.alerts.dummy.rst @@ -2,6 +2,6 @@ awslimitchecker.alerts.dummy module =================================== .. automodule:: awslimitchecker.alerts.dummy - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.alerts.pagerdutyv1.rst b/docs/source/awslimitchecker.alerts.pagerdutyv1.rst index 883f93b5..30c859c0 100644 --- a/docs/source/awslimitchecker.alerts.pagerdutyv1.rst +++ b/docs/source/awslimitchecker.alerts.pagerdutyv1.rst @@ -2,6 +2,6 @@ awslimitchecker.alerts.pagerdutyv1 module ========================================= .. automodule:: awslimitchecker.alerts.pagerdutyv1 - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.alerts.rst b/docs/source/awslimitchecker.alerts.rst index 0a91cbad..886467d8 100644 --- a/docs/source/awslimitchecker.alerts.rst +++ b/docs/source/awslimitchecker.alerts.rst @@ -2,9 +2,9 @@ awslimitchecker.alerts package ============================== .. automodule:: awslimitchecker.alerts - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: Submodules ---------- @@ -14,4 +14,3 @@ Submodules awslimitchecker.alerts.base awslimitchecker.alerts.dummy awslimitchecker.alerts.pagerdutyv1 - diff --git a/docs/source/awslimitchecker.checker.rst b/docs/source/awslimitchecker.checker.rst index d38a894a..bcb832db 100644 --- a/docs/source/awslimitchecker.checker.rst +++ b/docs/source/awslimitchecker.checker.rst @@ -2,6 +2,6 @@ awslimitchecker.checker module ============================== .. automodule:: awslimitchecker.checker - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.connectable.rst b/docs/source/awslimitchecker.connectable.rst index cd868f42..efbf5f69 100644 --- a/docs/source/awslimitchecker.connectable.rst +++ b/docs/source/awslimitchecker.connectable.rst @@ -2,6 +2,6 @@ awslimitchecker.connectable module ================================== .. automodule:: awslimitchecker.connectable - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.limit.rst b/docs/source/awslimitchecker.limit.rst index 48837510..93f5d406 100644 --- a/docs/source/awslimitchecker.limit.rst +++ b/docs/source/awslimitchecker.limit.rst @@ -2,6 +2,6 @@ awslimitchecker.limit module ============================ .. automodule:: awslimitchecker.limit - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.base.rst b/docs/source/awslimitchecker.metrics.base.rst index c80bc5f5..a5da667f 100644 --- a/docs/source/awslimitchecker.metrics.base.rst +++ b/docs/source/awslimitchecker.metrics.base.rst @@ -2,6 +2,6 @@ awslimitchecker.metrics.base module =================================== .. automodule:: awslimitchecker.metrics.base - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.datadog.rst b/docs/source/awslimitchecker.metrics.datadog.rst index 7977c7c2..41cd689e 100644 --- a/docs/source/awslimitchecker.metrics.datadog.rst +++ b/docs/source/awslimitchecker.metrics.datadog.rst @@ -2,6 +2,6 @@ awslimitchecker.metrics.datadog module ====================================== .. automodule:: awslimitchecker.metrics.datadog - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.dummy.rst b/docs/source/awslimitchecker.metrics.dummy.rst index c25b1c7c..d555e35a 100644 --- a/docs/source/awslimitchecker.metrics.dummy.rst +++ b/docs/source/awslimitchecker.metrics.dummy.rst @@ -2,6 +2,6 @@ awslimitchecker.metrics.dummy module ==================================== .. automodule:: awslimitchecker.metrics.dummy - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.metrics.rst b/docs/source/awslimitchecker.metrics.rst index 92f58ed7..33994cee 100644 --- a/docs/source/awslimitchecker.metrics.rst +++ b/docs/source/awslimitchecker.metrics.rst @@ -2,9 +2,9 @@ awslimitchecker.metrics package =============================== .. automodule:: awslimitchecker.metrics - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: Submodules ---------- @@ -14,4 +14,3 @@ Submodules awslimitchecker.metrics.base awslimitchecker.metrics.datadog awslimitchecker.metrics.dummy - diff --git a/docs/source/awslimitchecker.quotas.rst b/docs/source/awslimitchecker.quotas.rst new file mode 100644 index 00000000..3414ec31 --- /dev/null +++ b/docs/source/awslimitchecker.quotas.rst @@ -0,0 +1,7 @@ +awslimitchecker.quotas module +============================= + +.. automodule:: awslimitchecker.quotas + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.rst b/docs/source/awslimitchecker.rst index 07152c38..fb98f738 100644 --- a/docs/source/awslimitchecker.rst +++ b/docs/source/awslimitchecker.rst @@ -2,18 +2,18 @@ awslimitchecker package ======================= .. automodule:: awslimitchecker - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: Subpackages ----------- .. toctree:: - awslimitchecker.alerts - awslimitchecker.metrics - awslimitchecker.services + awslimitchecker.alerts + awslimitchecker.metrics + awslimitchecker.services Submodules ---------- @@ -23,8 +23,8 @@ Submodules awslimitchecker.checker awslimitchecker.connectable awslimitchecker.limit + awslimitchecker.quotas awslimitchecker.runner awslimitchecker.trustedadvisor awslimitchecker.utils awslimitchecker.version - diff --git a/docs/source/awslimitchecker.runner.rst b/docs/source/awslimitchecker.runner.rst index eba058c5..2c0d0a83 100644 --- a/docs/source/awslimitchecker.runner.rst +++ b/docs/source/awslimitchecker.runner.rst @@ -2,6 +2,6 @@ awslimitchecker.runner module ============================= .. automodule:: awslimitchecker.runner - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.apigateway.rst b/docs/source/awslimitchecker.services.apigateway.rst index c63bab2f..e83e03ee 100644 --- a/docs/source/awslimitchecker.services.apigateway.rst +++ b/docs/source/awslimitchecker.services.apigateway.rst @@ -2,6 +2,6 @@ awslimitchecker.services.apigateway module ========================================== .. automodule:: awslimitchecker.services.apigateway - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.autoscaling.rst b/docs/source/awslimitchecker.services.autoscaling.rst index 703edaab..cbbbf50e 100644 --- a/docs/source/awslimitchecker.services.autoscaling.rst +++ b/docs/source/awslimitchecker.services.autoscaling.rst @@ -2,6 +2,6 @@ awslimitchecker.services.autoscaling module =========================================== .. automodule:: awslimitchecker.services.autoscaling - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.base.rst b/docs/source/awslimitchecker.services.base.rst index 74597e6c..7d740a2f 100644 --- a/docs/source/awslimitchecker.services.base.rst +++ b/docs/source/awslimitchecker.services.base.rst @@ -2,6 +2,6 @@ awslimitchecker.services.base module ==================================== .. automodule:: awslimitchecker.services.base - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.cloudformation.rst b/docs/source/awslimitchecker.services.cloudformation.rst index 608ae898..cbeffe20 100644 --- a/docs/source/awslimitchecker.services.cloudformation.rst +++ b/docs/source/awslimitchecker.services.cloudformation.rst @@ -2,6 +2,6 @@ awslimitchecker.services.cloudformation module ============================================== .. automodule:: awslimitchecker.services.cloudformation - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.cloudtrail.rst b/docs/source/awslimitchecker.services.cloudtrail.rst index 25c989b6..8bd79ff7 100644 --- a/docs/source/awslimitchecker.services.cloudtrail.rst +++ b/docs/source/awslimitchecker.services.cloudtrail.rst @@ -2,6 +2,6 @@ awslimitchecker.services.cloudtrail module ========================================== .. automodule:: awslimitchecker.services.cloudtrail - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.directoryservice.rst b/docs/source/awslimitchecker.services.directoryservice.rst index 2dfe05d8..5e5a0930 100644 --- a/docs/source/awslimitchecker.services.directoryservice.rst +++ b/docs/source/awslimitchecker.services.directoryservice.rst @@ -2,6 +2,6 @@ awslimitchecker.services.directoryservice module ================================================ .. automodule:: awslimitchecker.services.directoryservice - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.dynamodb.rst b/docs/source/awslimitchecker.services.dynamodb.rst index 16eaf89a..fe6bd787 100644 --- a/docs/source/awslimitchecker.services.dynamodb.rst +++ b/docs/source/awslimitchecker.services.dynamodb.rst @@ -2,6 +2,6 @@ awslimitchecker.services.dynamodb module ======================================== .. automodule:: awslimitchecker.services.dynamodb - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.ebs.rst b/docs/source/awslimitchecker.services.ebs.rst index b6ebeaee..84fed438 100644 --- a/docs/source/awslimitchecker.services.ebs.rst +++ b/docs/source/awslimitchecker.services.ebs.rst @@ -2,6 +2,6 @@ awslimitchecker.services.ebs module =================================== .. automodule:: awslimitchecker.services.ebs - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.ec2.rst b/docs/source/awslimitchecker.services.ec2.rst index 95c09d8d..0ad5df17 100644 --- a/docs/source/awslimitchecker.services.ec2.rst +++ b/docs/source/awslimitchecker.services.ec2.rst @@ -2,6 +2,6 @@ awslimitchecker.services.ec2 module =================================== .. automodule:: awslimitchecker.services.ec2 - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.ecs.rst b/docs/source/awslimitchecker.services.ecs.rst index 5d23f901..64181c4d 100644 --- a/docs/source/awslimitchecker.services.ecs.rst +++ b/docs/source/awslimitchecker.services.ecs.rst @@ -2,6 +2,6 @@ awslimitchecker.services.ecs module =================================== .. automodule:: awslimitchecker.services.ecs - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.efs.rst b/docs/source/awslimitchecker.services.efs.rst index b2fc7b81..a0d80cb4 100644 --- a/docs/source/awslimitchecker.services.efs.rst +++ b/docs/source/awslimitchecker.services.efs.rst @@ -2,6 +2,6 @@ awslimitchecker.services.efs module =================================== .. automodule:: awslimitchecker.services.efs - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.elasticache.rst b/docs/source/awslimitchecker.services.elasticache.rst index 6b181166..390ccaa8 100644 --- a/docs/source/awslimitchecker.services.elasticache.rst +++ b/docs/source/awslimitchecker.services.elasticache.rst @@ -2,6 +2,6 @@ awslimitchecker.services.elasticache module =========================================== .. automodule:: awslimitchecker.services.elasticache - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.elasticbeanstalk.rst b/docs/source/awslimitchecker.services.elasticbeanstalk.rst index f5abf84f..55b414f2 100644 --- a/docs/source/awslimitchecker.services.elasticbeanstalk.rst +++ b/docs/source/awslimitchecker.services.elasticbeanstalk.rst @@ -2,6 +2,6 @@ awslimitchecker.services.elasticbeanstalk module ================================================ .. automodule:: awslimitchecker.services.elasticbeanstalk - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.elb.rst b/docs/source/awslimitchecker.services.elb.rst index 6f3be91a..6666a388 100644 --- a/docs/source/awslimitchecker.services.elb.rst +++ b/docs/source/awslimitchecker.services.elb.rst @@ -2,6 +2,6 @@ awslimitchecker.services.elb module =================================== .. automodule:: awslimitchecker.services.elb - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.firehose.rst b/docs/source/awslimitchecker.services.firehose.rst index 2b09e254..f7ef9476 100644 --- a/docs/source/awslimitchecker.services.firehose.rst +++ b/docs/source/awslimitchecker.services.firehose.rst @@ -2,6 +2,6 @@ awslimitchecker.services.firehose module ======================================== .. automodule:: awslimitchecker.services.firehose - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.iam.rst b/docs/source/awslimitchecker.services.iam.rst index 19a96609..183a938e 100644 --- a/docs/source/awslimitchecker.services.iam.rst +++ b/docs/source/awslimitchecker.services.iam.rst @@ -2,6 +2,6 @@ awslimitchecker.services.iam module =================================== .. automodule:: awslimitchecker.services.iam - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.lambdafunc.rst b/docs/source/awslimitchecker.services.lambdafunc.rst index df01e1a6..8bbf2cda 100644 --- a/docs/source/awslimitchecker.services.lambdafunc.rst +++ b/docs/source/awslimitchecker.services.lambdafunc.rst @@ -2,6 +2,6 @@ awslimitchecker.services.lambdafunc module ========================================== .. automodule:: awslimitchecker.services.lambdafunc - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.rds.rst b/docs/source/awslimitchecker.services.rds.rst index f49e3488..68222248 100644 --- a/docs/source/awslimitchecker.services.rds.rst +++ b/docs/source/awslimitchecker.services.rds.rst @@ -2,6 +2,6 @@ awslimitchecker.services.rds module =================================== .. automodule:: awslimitchecker.services.rds - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.redshift.rst b/docs/source/awslimitchecker.services.redshift.rst index 854134d6..a32a11ac 100644 --- a/docs/source/awslimitchecker.services.redshift.rst +++ b/docs/source/awslimitchecker.services.redshift.rst @@ -2,6 +2,6 @@ awslimitchecker.services.redshift module ======================================== .. automodule:: awslimitchecker.services.redshift - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.route53.rst b/docs/source/awslimitchecker.services.route53.rst index 455e1c94..b3bf7c38 100644 --- a/docs/source/awslimitchecker.services.route53.rst +++ b/docs/source/awslimitchecker.services.route53.rst @@ -2,6 +2,6 @@ awslimitchecker.services.route53 module ======================================= .. automodule:: awslimitchecker.services.route53 - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.rst b/docs/source/awslimitchecker.services.rst index 1615225f..d6281e1a 100644 --- a/docs/source/awslimitchecker.services.rst +++ b/docs/source/awslimitchecker.services.rst @@ -2,9 +2,9 @@ awslimitchecker.services package ================================ .. automodule:: awslimitchecker.services - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: Submodules ---------- @@ -34,4 +34,3 @@ Submodules awslimitchecker.services.s3 awslimitchecker.services.ses awslimitchecker.services.vpc - diff --git a/docs/source/awslimitchecker.services.s3.rst b/docs/source/awslimitchecker.services.s3.rst index 6d279e7e..4017e2f5 100644 --- a/docs/source/awslimitchecker.services.s3.rst +++ b/docs/source/awslimitchecker.services.s3.rst @@ -2,6 +2,6 @@ awslimitchecker.services.s3 module ================================== .. automodule:: awslimitchecker.services.s3 - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.ses.rst b/docs/source/awslimitchecker.services.ses.rst index 1804279d..071bfa0f 100644 --- a/docs/source/awslimitchecker.services.ses.rst +++ b/docs/source/awslimitchecker.services.ses.rst @@ -2,6 +2,6 @@ awslimitchecker.services.ses module =================================== .. automodule:: awslimitchecker.services.ses - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.services.vpc.rst b/docs/source/awslimitchecker.services.vpc.rst index eb9f6550..45a3563d 100644 --- a/docs/source/awslimitchecker.services.vpc.rst +++ b/docs/source/awslimitchecker.services.vpc.rst @@ -2,6 +2,6 @@ awslimitchecker.services.vpc module =================================== .. automodule:: awslimitchecker.services.vpc - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.trustedadvisor.rst b/docs/source/awslimitchecker.trustedadvisor.rst index a873a321..c1099535 100644 --- a/docs/source/awslimitchecker.trustedadvisor.rst +++ b/docs/source/awslimitchecker.trustedadvisor.rst @@ -2,6 +2,6 @@ awslimitchecker.trustedadvisor module ===================================== .. automodule:: awslimitchecker.trustedadvisor - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.utils.rst b/docs/source/awslimitchecker.utils.rst index bc379b98..7e36b5c9 100644 --- a/docs/source/awslimitchecker.utils.rst +++ b/docs/source/awslimitchecker.utils.rst @@ -2,6 +2,6 @@ awslimitchecker.utils module ============================ .. automodule:: awslimitchecker.utils - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/awslimitchecker.version.rst b/docs/source/awslimitchecker.version.rst index 26b5a987..3b7937a7 100644 --- a/docs/source/awslimitchecker.version.rst +++ b/docs/source/awslimitchecker.version.rst @@ -2,6 +2,6 @@ awslimitchecker.version module ============================== .. automodule:: awslimitchecker.version - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/cli_usage.rst b/docs/source/cli_usage.rst index 03b2ef85..3d876e29 100644 --- a/docs/source/cli_usage.rst +++ b/docs/source/cli_usage.rst @@ -33,7 +33,9 @@ use as a Nagios-compatible plugin). [-C CRITICAL_THRESHOLD] [-P PROFILE_NAME] [-A STS_ACCOUNT_ID] [-R STS_ACCOUNT_ROLE] [-E EXTERNAL_ID] [-M MFA_SERIAL_NUMBER] [-T MFA_TOKEN] - [-r REGION] [--skip-ta] + [-r REGION] [--role-partition ROLE_PARTITION] + [--ta-api-region TA_API_REGION] [--skip-ta] + [--skip-quotas] [--ta-refresh-wait | --ta-refresh-trigger | --ta-refresh-older TA_REFRESH_OLDER] [--ta-refresh-timeout TA_REFRESH_TIMEOUT] [--no-color] [--no-check-version] [-v] [-V] @@ -102,8 +104,17 @@ use as a Nagios-compatible plugin). MFA Token to use when assuming a role via STS -r REGION, --region REGION AWS region name to connect to; required for STS + --role-partition ROLE_PARTITION + AWS partition name to use for account_role when + connecting via STS; see documentation for more + information (default: "aws") + --ta-api-region TA_API_REGION + Region to use for Trusted Advisor / Support API + (default: us-east-1) --skip-ta do not attempt to pull *any* information on limits from Trusted Advisor + --skip-quotas Do not attempt to connect to Service Quotas service or + use its data for current limits --ta-refresh-wait If applicable, refresh all Trusted Advisor limit- related checks, and wait for the refresh to complete before continuing. @@ -186,17 +197,17 @@ or Trusted Advisor data, run with ``--list-defaults``: .. code-block:: console (venv)$ awslimitchecker --list-defaults - ApiGateway/API keys per account 500 - ApiGateway/Client certificates per account 60 - ApiGateway/Custom authorizers per API 10 - ApiGateway/Documentation parts per API 2000 - ApiGateway/Edge APIs per account 120 + ApiGateway/API keys per account 500 + ApiGateway/Client certificates per account 60 + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 120 (...) - Lambda/Function Count None + Lambda/Function Count None (...) - VPC/Subnets per VPC 200 - VPC/VPCs 5 - VPC/Virtual private gateways 5 + VPC/Subnets per VPC 200 + VPC/VPCs 5 + VPC/Virtual private gateways 5 @@ -212,44 +223,69 @@ and limits followed by ``(API)`` have been obtained from the service's API. .. code-block:: console (venv)$ awslimitchecker -l - ApiGateway/API keys per account 500 - ApiGateway/Client certificates per account 60 - ApiGateway/Custom authorizers per API 10 - ApiGateway/Documentation parts per API 2000 - ApiGateway/Edge APIs per account 120 + ApiGateway/API keys per account 500.0 (Quotas) + ApiGateway/Client certificates per account 60.0 (Quotas) + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 300.0 (Quotas) (...) - AutoScaling/Auto Scaling groups 1500 (API) + AutoScaling/Auto Scaling groups 1500 (API) (...) - Lambda/Function Count None + Lambda/Function Count None (...) - VPC/Subnets per VPC 200 - VPC/VPCs 1000 (TA) - VPC/Virtual private gateways 5 + VPC/Subnets per VPC 200 + VPC/VPCs 1000.0 (Quotas) + VPC/Virtual private gateways 5 Disabling Trusted Advisor Checks ++++++++++++++++++++++++++++++++ +Using the ``--skip-quotas`` option will disable attempting to query limit information +from the Service Quotas service. + +.. code-block:: console + + (venv)$ awslimitchecker -l --skip-quotas + ApiGateway/API keys per account 500 + ApiGateway/Client certificates per account 60 + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 120 + (...) + AutoScaling/Auto Scaling groups 1500 (API) + (...) + Lambda/Function Count None + (...) + VPC/Subnets per VPC 200 + VPC/VPCs 1000 (TA) + VPC/Virtual private gateways 5 + + + +Disabling Service Quotas service +++++++++++++++++++++++++++++++++ + Using the ``--skip-ta`` option will disable attempting to query limit information from Trusted Advisor for all commands. .. code-block:: console (venv)$ awslimitchecker -l --skip-ta - ApiGateway/API keys per account 500 - ApiGateway/Client certificates per account 60 - ApiGateway/Custom authorizers per API 10 - ApiGateway/Documentation parts per API 2000 - ApiGateway/Edge APIs per account 120 + ApiGateway/API keys per account 500.0 (Quotas) + ApiGateway/Client certificates per account 60.0 (Quotas) + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 300.0 (Quotas) (...) - AutoScaling/Auto Scaling groups 1500 (API) + AutoScaling/Auto Scaling groups 1500 (API) (...) - Lambda/Function Count None + Lambda/Function Count None (...) - VPC/Subnets per VPC 200 - VPC/VPCs 5 - VPC/Virtual private gateways 5 + VPC/Subnets per VPC 200 + VPC/VPCs 1000.0 (Quotas) + VPC/Virtual private gateways 5 @@ -306,15 +342,15 @@ using their IDs). .. code-block:: console (venv)$ awslimitchecker -u - ApiGateway/API keys per account 14 - ApiGateway/APIs per account 98 - ApiGateway/Client certificates per account 2 - ApiGateway/Custom authorizers per API max: 0bdkl1u8vk=2 (0bdkl1u8vk=2, 0cyhj26jhb=2 (...) - ApiGateway/Documentation parts per API max: 0bdkl1u8vk=2 (0bdkl1u8vk=2, 0cyhj26jhb=2 (...) + ApiGateway/API keys per account 22 + ApiGateway/Client certificates per account 3 + ApiGateway/Custom authorizers per API max: 00e87qs7ci=2 (00e87qs (...) + ApiGateway/Documentation parts per API max: 00e87qs7ci=2 (00e87qs (...) + ApiGateway/Edge APIs per account 207 (...) - VPC/Subnets per VPC max: vpc-c89074a9=41 (vpc-ae7bc5cb=1, vpc-7bc (...) - VPC/VPCs 22 - VPC/Virtual private gateways 5 + VPC/Subnets per VPC max: vpc-c89074a9=41 (vpc- (...) + VPC/VPCs 39 + VPC/Virtual private gateways 4 @@ -339,17 +375,21 @@ For example, to override the limits of EC2's "EC2-Classic Elastic IPs" and .. code-block:: console (venv)$ awslimitchecker -L "AutoScaling/Auto Scaling groups"=321 --limit="AutoScaling/Launch configurations"=456 -l - ApiGateway/API keys per account 500 - ApiGateway/APIs per account 60 - ApiGateway/Client certificates per account 60 - ApiGateway/Custom authorizers per API 10 - ApiGateway/Documentation parts per API 2000 + ApiGateway/API keys per account 500.0 (Quotas) + ApiGateway/Client certificates per account 60.0 (Quotas) + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 300.0 (Quotas) (...) - CloudFormation/Stacks 2500 (API) + CloudFormation/Stacks 4000 (API) (...) - VPC/Subnets per VPC 200 - VPC/VPCs 1000 (TA) - VPC/Virtual private gateways 5 + Lambda/Function Count None + (...) + VPC/Subnets per VPC 200 + VPC/VPCs 1000.0 (Quotas) + VPC/Virtual private gateways 5 + + This example simply sets the overrides, and then prints the limits for confirmation. @@ -364,22 +404,22 @@ You could also set the same limit overrides using a JSON file stored at ``limit_ } } + Using a command like: .. code-block:: console (venv)$ awslimitchecker --limit-override-json=limit_overrides.json -l - ApiGateway/API keys per account 500 - ApiGateway/APIs per account 60 - ApiGateway/Client certificates per account 60 - ApiGateway/Custom authorizers per API 10 - ApiGateway/Documentation parts per API 2000 + ApiGateway/API keys per account 500.0 (Quotas) + ApiGateway/Client certificates per account 60.0 (Quotas) + ApiGateway/Custom authorizers per API 10 + ApiGateway/Documentation parts per API 2000 + ApiGateway/Edge APIs per account 300.0 (Quotas) (...) - CloudFormation/Stacks 2500 (API) - (...) - VPC/Subnets per VPC 200 - VPC/VPCs 1000 (TA) - VPC/Virtual private gateways 5 + VPC/Subnets per VPC 200 + VPC/VPCs 1000.0 (Quotas) + VPC/Virtual private gateways 5 + Check Limits Against Thresholds @@ -408,15 +448,17 @@ threshold only, and another has crossed the critical threshold): .. code-block:: console (venv)$ awslimitchecker --no-color - ApiGateway/APIs per account (limit 60) CRITICAL: 211 - DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: something-goes-here-index (...) - DynamoDB/Tables Per Region (limit 256) CRITICAL: 504 - EC2/Security groups per VPC (limit 500) CRITICAL: vpc-12345678=674, vpc-c (...) - EC2/VPC security groups per elastic network interface (limit 5) CRITICAL: eni-01234567890123456=5, (...) + CloudFormation/Stacks (limit 4000) WARNING: 3396 + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EBS/Active snapshots (limit 40000.0) WARNING: 33387 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-aaaaaaaa=50, sg-bbbb (...) (...) - VPC/Entries per route table (limit 50) WARNING: rtb-01234567=40, rtb-6789 (...) - VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=9, us-east-1c= (...) - VPC/Virtual private gateways (limit 5) CRITICAL: 6 + VPC/Entries per route table (limit 50) WARNING: rtb-aaaaaaaa=43, rtb-bbbb (...) + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) + VPC/Virtual private gateways (limit 5) WARNING: 4 + + .. _cli_usage.threshold_overrides: @@ -428,15 +470,16 @@ To set the warning threshold of 50% and a critical threshold of 75% when checkin .. code-block:: console (venv)$ awslimitchecker -W 97 --critical=98 --no-color - ApiGateway/APIs per account (limit 60) CRITICAL: 98 - DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: sale_setup_draft_vehicles (...) - DynamoDB/Tables Per Region (limit 256) WARNING: 250 - EC2/Security groups per VPC (limit 500) CRITICAL: vpc-c89074a9=863 - EC2/VPC security groups per elastic network interface (limit 5) CRITICAL: eni-8226ce61=5 + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name (...) + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-cccccccc=49, sg-eeeee (...) + EC2/Security groups per VPC (limit 500) CRITICAL: vpc-dddddddd=726, vpc-c (...) (...) - S3/Buckets (limit 100) CRITICAL: 657 - VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=7, us-east-1c= (...) - VPC/Virtual private gateways (limit 5) CRITICAL: 5 + RDS/VPC Security Groups (limit 5) CRITICAL: 5 + S3/Buckets (limit 100) CRITICAL: 946 + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) + + You can also set custom thresholds on a per-limit basis using the ``--threshold-override-json`` CLI option, which accepts the path to a JSON file @@ -484,15 +527,14 @@ Using a command like: .. code-block:: console (venv)$ awslimitchecker -W 97 --critical=98 --no-color --threshold-override-json=s3://bucketname/path/overrides.json - ApiGateway/APIs per account (limit 60) CRITICAL: 98 - DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: sale_setup_draft_vehicles (...) - DynamoDB/Tables Per Region (limit 256) WARNING: 250 - EC2/Security groups per VPC (limit 500) CRITICAL: vpc-c89074a9=863 - EC2/VPC security groups per elastic network interface (limit 5) CRITICAL: eni-8226ce61=5 + DynamoDB/Local Secondary Indexes (limit 5) CRITICAL: some_app_name_here=5 + DynamoDB/Tables Per Region (limit 256) CRITICAL: 554 + EC2/Rules per VPC security group (limit 50) CRITICAL: sg-aaaaaaaa=49, sg-bbbbb (...) + EC2/Security groups per VPC (limit 500) CRITICAL: vpc-cccccccc=726, vpc-c (...) (...) - S3/Buckets (limit 100) CRITICAL: 657 - VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=7, us-east-1c= (...) - VPC/Virtual private gateways (limit 5) CRITICAL: 5 + RDS/VPC Security Groups (limit 5) CRITICAL: 5 + S3/Buckets (limit 100) CRITICAL: 946 + VPC/NAT Gateways per AZ (limit 5) CRITICAL: us-east-1d=5, us-east-1c= (...) @@ -518,8 +560,8 @@ can be seen with the ``--list-metrics-providers`` option: The configuration options required by each metrics provider are specified in the providers' documentation: -* :py:class:`~awslimitchecker.metrics.datadog.Datadog` * :py:class:`~awslimitchecker.metrics.dummy.Dummy` +* :py:class:`~awslimitchecker.metrics.datadog.Datadog` For example, to use the :py:class:`~awslimitchecker.metrics.datadog.Datadog` @@ -558,8 +600,8 @@ of awslimitchecker can be seen with the ``--list-alert-providers`` option: The configuration options required by each alert provider are specified in the providers' documentation: -* :py:class:`~awslimitchecker.alerts.pagerdutyv1.PagerDutyV1` * :py:class:`~awslimitchecker.alerts.dummy.Dummy` +* :py:class:`~awslimitchecker.alerts.pagerdutyv1.PagerDutyV1` For example, to use the :py:class:`~awslimitchecker.alerts.pagerdutyv1.PagerDutyV1` @@ -636,3 +678,21 @@ If you also need to specify an ``external_id`` of "myid", you can do that with t Please note that this assumes that you already have STS configured and working between your account and the 123456789012 destination account; see the `documentation `_ for further information. + +.. _cli_usage.partitions: + +Partitions and Trusted Advisor Regions +++++++++++++++++++++++++++++++++++++++ + +awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``. + +.. _cli_usage.throttling: + +Handling Throttling and Rate Limiting ++++++++++++++++++++++++++++++++++++++ + +In some very large and busy AWS accounts, from time to time awslimitchecker might die on unhandled ``Throttling`` or ``RateExceeded`` exceptions. botocore, the underlying low-level AWS API client library that we use, automatically catches these exceptions and retries them up to a per-AWS-API default number of times (generally four for most APIs) with an exponential backoff. In very busy accounts, it may be desirable to increase the default number of retries. + +This can be accomplished on a per-API basis (where the API name is the ``service_name`` that would be sent to :py:meth:`boto3.session.Session.client` and is set as the :py:attr:`~.awslimitchecker.services.base._AwsService.api_name` attribute on each :py:class:`~.awslimitchecker.services.base._AwsService` subclass) by setting an environment variable ``BOTO_MAX_RETRIES_`` to the maximum number of attempts you'd like for that service. + +For example, if you have issues with rate limiting of the ``cloudformation:DescribeStacks`` still failing after the default of four attempts, and you'd like to use ten (10) attempts instead, you could ``export BOTO_MAX_RETRIES_cloudformation=10`` before running ``awslimitchecker``. diff --git a/docs/source/cli_usage.rst.template b/docs/source/cli_usage.rst.template index 0738b955..01cc4451 100644 --- a/docs/source/cli_usage.rst.template +++ b/docs/source/cli_usage.rst.template @@ -61,6 +61,14 @@ and limits followed by ``(API)`` have been obtained from the service's API. Disabling Trusted Advisor Checks ++++++++++++++++++++++++++++++++ +Using the ``--skip-quotas`` option will disable attempting to query limit information +from the Service Quotas service. + +{skip_quotas} + +Disabling Service Quotas service +++++++++++++++++++++++++++++++++ + Using the ``--skip-ta`` option will disable attempting to query limit information from Trusted Advisor for all commands. @@ -311,3 +319,21 @@ If you also need to specify an ``external_id`` of "myid", you can do that with t Please note that this assumes that you already have STS configured and working between your account and the 123456789012 destination account; see the `documentation `_ for further information. + +.. _cli_usage.partitions: + +Partitions and Trusted Advisor Regions +++++++++++++++++++++++++++++++++++++++ + +awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``--role-partition`` option to awslimitchecker, like ``--role-partition=aws-cn`` for the China (Beijing) partition. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``--ta-api-region`` option, like ``--ta-api-region=us-gov-west-1``. + +.. _cli_usage.throttling: + +Handling Throttling and Rate Limiting ++++++++++++++++++++++++++++++++++++++ + +In some very large and busy AWS accounts, from time to time awslimitchecker might die on unhandled ``Throttling`` or ``RateExceeded`` exceptions. botocore, the underlying low-level AWS API client library that we use, automatically catches these exceptions and retries them up to a per-AWS-API default number of times (generally four for most APIs) with an exponential backoff. In very busy accounts, it may be desirable to increase the default number of retries. + +This can be accomplished on a per-API basis (where the API name is the ``service_name`` that would be sent to :py:meth:`boto3.session.Session.client` and is set as the :py:attr:`~.awslimitchecker.services.base._AwsService.api_name` attribute on each :py:class:`~.awslimitchecker.services.base._AwsService` subclass) by setting an environment variable ``BOTO_MAX_RETRIES_`` to the maximum number of attempts you'd like for that service. + +For example, if you have issues with rate limiting of the ``cloudformation:DescribeStacks`` still failing after the default of four attempts, and you'd like to use ten (10) attempts instead, you could ``export BOTO_MAX_RETRIES_cloudformation=10`` before running ``awslimitchecker``. diff --git a/docs/source/development.rst b/docs/source/development.rst index 202bf87d..c432ab6e 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -49,7 +49,7 @@ To setup awslimitchecker for development: 1. Fork the `awslimitchecker `_ repository on GitHub -2. Create a ``virtualenv`` to run the code in: +2. Create a ``virtualenv`` (using Python 3.5 or later) to run the code in: .. code-block:: bash @@ -104,7 +104,7 @@ Adding New EC2 Instance Types the EC2 Pricing API that aren't present in awslimitchecker and output a list of them. 2. In ``services/ec2.py`` update the constants in :py:meth:`~._Ec2Service._instance_types` accordingly. 3. Check the `EC2 Instance Type limits page `__ - for any new types that have non-default limits, and update :py:meth:`~._Ec2Service._get_limits_instances` accordingly. + for any new types that have non-default limits, and update :py:meth:`~._Ec2Service._get_limits_instances_nonvcpu` accordingly. 4. Update ``tests/services/test_ec2.py`` as needed. .. _development.adding_checks: @@ -114,7 +114,8 @@ Adding New Limits and Checks to Existing Services First, note that all calls to boto3 client ("low-level") methods that return a dict response that can include 'NextToken' or another pagination marker, should be called through -:py:func:`~awslimitchecker.utils.paginate_dict` with the appropriate parameters. +:py:func:`~awslimitchecker.utils.paginate_dict` with the appropriate parameters +if the boto3 client can't paginate the call itself. 1. Add a new :py:class:`~.AwsLimit` instance to the return value of the Service class's :py:meth:`~._AwsService.get_limits` method. If Trusted Advisor @@ -130,7 +131,11 @@ include 'NextToken' or another pagination marker, should be called through include your new limit, ensure that this value is updated in the limit via its :py:meth:`~.AwsLimit._set_api_limit` method. This should be done in the Service class's ``_update_limits_from_api()`` method. -4. Ensure complete test coverage for the above. +4. If Service Quotas returns data for this limit, be sure that the parent + :py:class:`~._AwsService` class has its :py:attr:`~._AwsService.quotas_service_code` + attribute set appropriately and specify the ``quotas_name`` argument to the + :py:class:`~.AwsLimit` constructor if the quota name is different from the limit name. +5. Ensure complete test coverage for the above. In cases where the AWS service API has a different name than what is reported by Trusted Advisor, or legacy cases where Trusted Advisor support is retroactively @@ -179,19 +184,22 @@ include 'NextToken' or another pagination marker, should be called through 7. If your service has an API action to retrieve limit/quota information (i.e. ``DescribeAccountAttributes`` for EC2 and RDS), ensure that the service class has an ``_update_limits_from_api()`` method which makes this API call and updates each relevant AwsLimit via its :py:meth:`~.AwsLimit._set_api_limit` method. -8. Test your code; 100% test coverage is expected, and mocks should be using ``autospec`` or ``spec_set``. -9. Ensure the :py:meth:`~awslimitchecker.services.base._AwsService.required_iam_permissions` method of your new class - returns a list of all IAM permissions required for it to work. -10. Run all tox jobs, or at least one python version, docs and coverage. -11. Commit the updated documentation to the repository. -12. As there is no programmatic way to validate IAM policies, once you are done writing your service, grab the +8. If the Service Quotas service returns information on limits for your service, be sure you set the :py:attr:`~._AwsService.quotas_service_code` + attribute appropriately, and also pass the ``quota_name`` keyword argument to the constructor of any :py:class:`~.AwsLimit` classes + which have information available via Service Quotas. +9. Test your code; 100% test coverage is expected, and mocks should be using ``autospec`` or ``spec_set``. +10. Ensure the :py:meth:`~awslimitchecker.services.base._AwsService.required_iam_permissions` method of your new class + returns a list of all IAM permissions required for it to work. +11. Run all tox jobs, or at least one python version, docs and coverage. +12. Commit the updated documentation to the repository. +13. As there is no programmatic way to validate IAM policies, once you are done writing your service, grab the output of ``awslimitchecker --iam-policy``, login to your AWS account, and navigate to the IAM page. Click through to create a new policy, paste the output of the ``--iam-policy`` command, and click the "Validate Policy" button. Correct any errors that occur; for more information, see the AWS IAM docs on `Using Policy Validator `_. It would also be a good idea to run any policy changes through the `Policy Simulator `_. -13. Submit your pull request. +14. Submit your pull request. .. _development.adding_ta: @@ -372,8 +380,7 @@ For issues: Versioning Policy ----------------- -As of version 1.0.0, awslimitchecker strives to follow `semver 2.0.0 `_ -for versioning, with some specific clarifications: +As of version 1.0.0, awslimitchecker strives to follow `semver 2.0.0 `_ for versioning, with some specific clarifications: * Major version bumps (backwards-incompatible changes): diff --git a/docs/source/docker.rst b/docs/source/docker.rst index af994cf9..91fbcce8 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -44,91 +44,135 @@ command for :ref:`cli_usage`. For example, to show help: $ docker run jantman/awslimitchecker --help usage: awslimitchecker [-h] [-S [SERVICE [SERVICE ...]]] - [--skip-service SKIP_SERVICE] [--skip-check SKIP_CHECK] - [-s] [-l] [--list-defaults] [-L LIMIT] [-u] - [--iam-policy] [-W WARNING_THRESHOLD] - [-C CRITICAL_THRESHOLD] [-P PROFILE_NAME] - [-A STS_ACCOUNT_ID] [-R STS_ACCOUNT_ROLE] - [-E EXTERNAL_ID] [-M MFA_SERIAL_NUMBER] [-T MFA_TOKEN] - [-r REGION] [--skip-ta] - [--ta-refresh-wait | --ta-refresh-trigger | --ta-refresh-older TA_REFRESH_OLDER] - [--ta-refresh-timeout TA_REFRESH_TIMEOUT] [--no-color] - [--no-check-version] [-v] [-V] + [--skip-service SKIP_SERVICE] [--skip-check SKIP_CHECK] + [-s] [-l] [--list-defaults] [-L LIMIT] + [--limit-override-json LIMIT_OVERRIDE_JSON] + [--threshold-override-json THRESHOLD_OVERRIDE_JSON] + [-u] [--iam-policy] [-W WARNING_THRESHOLD] + [-C CRITICAL_THRESHOLD] [-P PROFILE_NAME] + [-A STS_ACCOUNT_ID] [-R STS_ACCOUNT_ROLE] + [-E EXTERNAL_ID] [-M MFA_SERIAL_NUMBER] [-T MFA_TOKEN] + [-r REGION] [--role-partition ROLE_PARTITION] + [--ta-api-region TA_API_REGION] [--skip-ta] + [--skip-quotas] + [--ta-refresh-wait | --ta-refresh-trigger | --ta-refresh-older TA_REFRESH_OLDER] + [--ta-refresh-timeout TA_REFRESH_TIMEOUT] [--no-color] + [--no-check-version] [-v] [-V] + [--list-metrics-providers] + [--metrics-provider METRICS_PROVIDER] + [--metrics-config METRICS_CONFIG] + [--list-alert-providers] + [--alert-provider ALERT_PROVIDER] + [--alert-config ALERT_CONFIG] Report on AWS service limits and usage via boto3, optionally warn about any services with usage nearing or exceeding their limits. For further help, see optional arguments: - -h, --help show this help message and exit - -S [SERVICE [SERVICE ...]], --service [SERVICE [SERVICE ...]] - perform action for only the specified service name; - see -s|--list-services for valid names - --skip-service SKIP_SERVICE - avoid performing actions for the specified service - name; see -s|--list-services for valid names - --skip-check SKIP_CHECK - avoid performing actions for the specified check name - -s, --list-services print a list of all AWS service types that - awslimitchecker knows how to check - -l, --list-limits print all AWS effective limits in - "service_name/limit_name" format - --list-defaults print all AWS default limits in - "service_name/limit_name" format - -L LIMIT, --limit LIMIT - override a single AWS limit, specified in - "service_name/limit_name=value" format; can be - specified multiple times. - -u, --show-usage find and print the current usage of all AWS services - with known limits - --iam-policy output a JSON serialized IAM Policy listing the - required permissions for awslimitchecker to run - correctly. - -W WARNING_THRESHOLD, --warning-threshold WARNING_THRESHOLD - default warning threshold (percentage of limit); - default: 80 - -C CRITICAL_THRESHOLD, --critical-threshold CRITICAL_THRESHOLD - default critical threshold (percentage of limit); - default: 99 - -P PROFILE_NAME, --profile PROFILE_NAME - Name of profile in the AWS cross-sdk credentials file - to use credentials from; similar to the corresponding - awscli option - -A STS_ACCOUNT_ID, --sts-account-id STS_ACCOUNT_ID - for use with STS, the Account ID of the destination - account (account to assume a role in) - -R STS_ACCOUNT_ROLE, --sts-account-role STS_ACCOUNT_ROLE - for use with STS, the name of the IAM role to assume - -E EXTERNAL_ID, --external-id EXTERNAL_ID - External ID to use when assuming a role via STS - -M MFA_SERIAL_NUMBER, --mfa-serial-number MFA_SERIAL_NUMBER - MFA Serial Number to use when assuming a role via STS - -T MFA_TOKEN, --mfa-token MFA_TOKEN - MFA Token to use when assuming a role via STS - -r REGION, --region REGION - AWS region name to connect to; required for STS - --skip-ta do not attempt to pull *any* information on limits - from Trusted Advisor - --ta-refresh-wait If applicable, refresh all Trusted Advisor limit- - related checks, and wait for the refresh to complete - before continuing. - --ta-refresh-trigger If applicable, trigger refreshes for all Trusted - Advisor limit-related checks, but do not wait for them - to finish refreshing; trigger the refresh and continue - on (useful to ensure checks are refreshed before the - next scheduled run). - --ta-refresh-older TA_REFRESH_OLDER - If applicable, trigger refreshes for all Trusted - Advisor limit-related checks with results more than - this number of seconds old. Wait for the refresh to - complete before continuing. - --ta-refresh-timeout TA_REFRESH_TIMEOUT - If waiting for TA checks to refresh, wait up to this - number of seconds before continuing on anyway. - --no-color do not colorize output - --no-check-version do not check latest version at startup - -v, --verbose verbose output. specify twice for debug-level output. - -V, --version print version number and exit. + -h, --help show this help message and exit + -S [SERVICE [SERVICE ...]], --service [SERVICE [SERVICE ...]] + perform action for only the specified service name; + see -s|--list-services for valid names + --skip-service SKIP_SERVICE + avoid performing actions for the specified service + name; see -s|--list-services for valid names + --skip-check SKIP_CHECK + avoid performing actions for the specified check name + -s, --list-services print a list of all AWS service types that + awslimitchecker knows how to check + -l, --list-limits print all AWS effective limits in + "service_name/limit_name" format + --list-defaults print all AWS default limits in + "service_name/limit_name" format + -L LIMIT, --limit LIMIT + override a single AWS limit, specified in + "service_name/limit_name=value" format; can be + specified multiple times. + --limit-override-json LIMIT_OVERRIDE_JSON + Absolute or relative path, or s3:// URL, to a JSON + file specifying limit overrides. See docs for expected + format. + --threshold-override-json THRESHOLD_OVERRIDE_JSON + Absolute or relative path, or s3:// URL, to a JSON + file specifying threshold overrides. See docs for + expected format. + -u, --show-usage find and print the current usage of all AWS services + with known limits + --iam-policy output a JSON serialized IAM Policy listing the + required permissions for awslimitchecker to run + correctly. + -W WARNING_THRESHOLD, --warning-threshold WARNING_THRESHOLD + default warning threshold (percentage of limit); + default: 80 + -C CRITICAL_THRESHOLD, --critical-threshold CRITICAL_THRESHOLD + default critical threshold (percentage of limit); + default: 99 + -P PROFILE_NAME, --profile PROFILE_NAME + Name of profile in the AWS cross-sdk credentials file + to use credentials from; similar to the corresponding + awscli option + -A STS_ACCOUNT_ID, --sts-account-id STS_ACCOUNT_ID + for use with STS, the Account ID of the destination + account (account to assume a role in) + -R STS_ACCOUNT_ROLE, --sts-account-role STS_ACCOUNT_ROLE + for use with STS, the name of the IAM role to assume + -E EXTERNAL_ID, --external-id EXTERNAL_ID + External ID to use when assuming a role via STS + -M MFA_SERIAL_NUMBER, --mfa-serial-number MFA_SERIAL_NUMBER + MFA Serial Number to use when assuming a role via STS + -T MFA_TOKEN, --mfa-token MFA_TOKEN + MFA Token to use when assuming a role via STS + -r REGION, --region REGION + AWS region name to connect to; required for STS + --role-partition ROLE_PARTITION + AWS partition name to use for account_role when + connecting via STS; see documentation for more + information (default: "aws") + --ta-api-region TA_API_REGION + Region to use for Trusted Advisor / Support API + (default: us-east-1) + --skip-ta do not attempt to pull *any* information on limits + from Trusted Advisor + --skip-quotas Do not attempt to connect to Service Quotas service or + use its data for current limits + --ta-refresh-wait If applicable, refresh all Trusted Advisor limit- + related checks, and wait for the refresh to complete + before continuing. + --ta-refresh-trigger If applicable, trigger refreshes for all Trusted + Advisor limit-related checks, but do not wait for them + to finish refreshing; trigger the refresh and continue + on (useful to ensure checks are refreshed before the + next scheduled run). + --ta-refresh-older TA_REFRESH_OLDER + If applicable, trigger refreshes for all Trusted + Advisor limit-related checks with results more than + this number of seconds old. Wait for the refresh to + complete before continuing. + --ta-refresh-timeout TA_REFRESH_TIMEOUT + If waiting for TA checks to refresh, wait up to this + number of seconds before continuing on anyway. + --no-color do not colorize output + --no-check-version do not check latest version at startup + -v, --verbose verbose output. specify twice for debug-level output. + -V, --version print version number and exit. + --list-metrics-providers + List available metrics providers and exit + --metrics-provider METRICS_PROVIDER + Metrics provider class name, to enable sending metrics + --metrics-config METRICS_CONFIG + Specify key/value parameters for the metrics provider + constructor. See documentation for further + information. + --list-alert-providers + List available alert providers and exit + --alert-provider ALERT_PROVIDER + Alert provider class name, to enable sending + notifications + --alert-config ALERT_CONFIG + Specify key/value parameters for the alert provider + constructor. See documentation for further + information. awslimitchecker is AGPLv3-licensed Free Software. Anyone using this program, even remotely over a network, is entitled to a copy of the source code. Use diff --git a/docs/source/ec2_nonvcpu_limits.txt b/docs/source/ec2_nonvcpu_limits.txt new file mode 100644 index 00000000..b2398b3d --- /dev/null +++ b/docs/source/ec2_nonvcpu_limits.txt @@ -0,0 +1,283 @@ +================================================= =============== ======= ==== +Limit Trusted Advisor API Default +================================================= =============== ======= ==== +Elastic IP addresses (EIPs) |check| |check| 5 +Max active spot fleets per region 1000 +Max launch specifications per spot fleet 50 +Max spot instance requests per region 20 +Max target capacity for all spot fleets in region 5000 +Max target capacity per spot fleet 3000 +Rules per VPC security group 50 +Running On-Demand EC2 instances |check| 20 +Running On-Demand a1.2xlarge instances 20 +Running On-Demand a1.4xlarge instances 20 +Running On-Demand a1.large instances 20 +Running On-Demand a1.medium instances 20 +Running On-Demand a1.metal instances 20 +Running On-Demand a1.xlarge instances 20 +Running On-Demand c1.medium instances 20 +Running On-Demand c1.xlarge instances 20 +Running On-Demand c3.2xlarge instances 20 +Running On-Demand c3.4xlarge instances 20 +Running On-Demand c3.8xlarge instances 20 +Running On-Demand c3.large instances |check| 20 +Running On-Demand c3.xlarge instances |check| 20 +Running On-Demand c4.2xlarge instances |check| 20 +Running On-Demand c4.4xlarge instances |check| 10 +Running On-Demand c4.8xlarge instances |check| 5 +Running On-Demand c4.large instances |check| 20 +Running On-Demand c4.xlarge instances |check| 20 +Running On-Demand c5.12xlarge instances 20 +Running On-Demand c5.18xlarge instances 5 +Running On-Demand c5.24xlarge instances 20 +Running On-Demand c5.2xlarge instances |check| 20 +Running On-Demand c5.4xlarge instances |check| 10 +Running On-Demand c5.9xlarge instances 5 +Running On-Demand c5.large instances |check| 20 +Running On-Demand c5.metal instances 20 +Running On-Demand c5.xlarge instances 20 +Running On-Demand c5d.12xlarge instances 20 +Running On-Demand c5d.18xlarge instances 20 +Running On-Demand c5d.24xlarge instances 20 +Running On-Demand c5d.2xlarge instances 20 +Running On-Demand c5d.4xlarge instances 20 +Running On-Demand c5d.9xlarge instances 20 +Running On-Demand c5d.large instances 20 +Running On-Demand c5d.metal instances 20 +Running On-Demand c5d.xlarge instances 20 +Running On-Demand c5n.18xlarge instances 20 +Running On-Demand c5n.2xlarge instances 20 +Running On-Demand c5n.4xlarge instances 20 +Running On-Demand c5n.9xlarge instances 20 +Running On-Demand c5n.large instances 20 +Running On-Demand c5n.metal instances 20 +Running On-Demand c5n.xlarge instances 20 +Running On-Demand cc1.4xlarge instances 20 +Running On-Demand cc2.8xlarge instances 20 +Running On-Demand cg1.4xlarge instances 2 +Running On-Demand cr1.8xlarge instances 2 +Running On-Demand d2.2xlarge instances 20 +Running On-Demand d2.4xlarge instances 10 +Running On-Demand d2.8xlarge instances 5 +Running On-Demand d2.xlarge instances 20 +Running On-Demand f1.16xlarge instances 20 +Running On-Demand f1.2xlarge instances 20 +Running On-Demand f1.4xlarge instances 20 +Running On-Demand g2.2xlarge instances 5 +Running On-Demand g2.8xlarge instances 2 +Running On-Demand g3.16xlarge instances 1 +Running On-Demand g3.4xlarge instances 1 +Running On-Demand g3.8xlarge instances 1 +Running On-Demand g3s.xlarge instances 20 +Running On-Demand g4dn.12xlarge instances 20 +Running On-Demand g4dn.16xlarge instances 20 +Running On-Demand g4dn.2xlarge instances 20 +Running On-Demand g4dn.4xlarge instances 20 +Running On-Demand g4dn.8xlarge instances 20 +Running On-Demand g4dn.metal instances 20 +Running On-Demand g4dn.xlarge instances 20 +Running On-Demand h1.16xlarge instances 5 +Running On-Demand h1.2xlarge instances 20 +Running On-Demand h1.4xlarge instances 20 +Running On-Demand h1.8xlarge instances 10 +Running On-Demand hi1.4xlarge instances 2 +Running On-Demand hs1.8xlarge instances 2 +Running On-Demand i2.2xlarge instances 8 +Running On-Demand i2.4xlarge instances 4 +Running On-Demand i2.8xlarge instances 2 +Running On-Demand i2.xlarge instances 8 +Running On-Demand i3.16xlarge instances 2 +Running On-Demand i3.2xlarge instances 2 +Running On-Demand i3.4xlarge instances 2 +Running On-Demand i3.8xlarge instances 2 +Running On-Demand i3.large instances 2 +Running On-Demand i3.metal instances 20 +Running On-Demand i3.xlarge instances 2 +Running On-Demand i3en.12xlarge instances 20 +Running On-Demand i3en.24xlarge instances 20 +Running On-Demand i3en.2xlarge instances 20 +Running On-Demand i3en.3xlarge instances 20 +Running On-Demand i3en.6xlarge instances 20 +Running On-Demand i3en.large instances 20 +Running On-Demand i3en.xlarge instances 20 +Running On-Demand m1.large instances 20 +Running On-Demand m1.medium instances 20 +Running On-Demand m1.small instances |check| 20 +Running On-Demand m1.xlarge instances 20 +Running On-Demand m2.2xlarge instances 20 +Running On-Demand m2.4xlarge instances 20 +Running On-Demand m2.xlarge instances 20 +Running On-Demand m3.2xlarge instances |check| 20 +Running On-Demand m3.large instances |check| 20 +Running On-Demand m3.medium instances |check| 20 +Running On-Demand m3.xlarge instances |check| 20 +Running On-Demand m4.10xlarge instances 5 +Running On-Demand m4.16xlarge instances |check| 5 +Running On-Demand m4.2xlarge instances |check| 20 +Running On-Demand m4.4xlarge instances |check| 10 +Running On-Demand m4.large instances |check| 20 +Running On-Demand m4.xlarge instances |check| 20 +Running On-Demand m5.12xlarge instances 5 +Running On-Demand m5.16xlarge instances 20 +Running On-Demand m5.24xlarge instances 5 +Running On-Demand m5.2xlarge instances 20 +Running On-Demand m5.4xlarge instances 10 +Running On-Demand m5.8xlarge instances 20 +Running On-Demand m5.large instances |check| 20 +Running On-Demand m5.metal instances 20 +Running On-Demand m5.xlarge instances 20 +Running On-Demand m5a.12xlarge instances 20 +Running On-Demand m5a.16xlarge instances 20 +Running On-Demand m5a.24xlarge instances 20 +Running On-Demand m5a.2xlarge instances 20 +Running On-Demand m5a.4xlarge instances 20 +Running On-Demand m5a.8xlarge instances 20 +Running On-Demand m5a.large instances 20 +Running On-Demand m5a.xlarge instances 20 +Running On-Demand m5ad.12xlarge instances 20 +Running On-Demand m5ad.16xlarge instances 20 +Running On-Demand m5ad.24xlarge instances 20 +Running On-Demand m5ad.2xlarge instances 20 +Running On-Demand m5ad.4xlarge instances 20 +Running On-Demand m5ad.8xlarge instances 20 +Running On-Demand m5ad.large instances 20 +Running On-Demand m5ad.xlarge instances 20 +Running On-Demand m5d.12xlarge instances 20 +Running On-Demand m5d.16xlarge instances 20 +Running On-Demand m5d.24xlarge instances 20 +Running On-Demand m5d.2xlarge instances 20 +Running On-Demand m5d.4xlarge instances 20 +Running On-Demand m5d.8xlarge instances 20 +Running On-Demand m5d.large instances 20 +Running On-Demand m5d.metal instances 20 +Running On-Demand m5d.xlarge instances 20 +Running On-Demand m5dn.12xlarge instances 20 +Running On-Demand m5dn.16xlarge instances 20 +Running On-Demand m5dn.24xlarge instances 20 +Running On-Demand m5dn.2xlarge instances 20 +Running On-Demand m5dn.4xlarge instances 20 +Running On-Demand m5dn.8xlarge instances 20 +Running On-Demand m5dn.large instances 20 +Running On-Demand m5dn.metal instances 20 +Running On-Demand m5dn.xlarge instances 20 +Running On-Demand m5n.12xlarge instances 20 +Running On-Demand m5n.16xlarge instances 20 +Running On-Demand m5n.24xlarge instances 20 +Running On-Demand m5n.2xlarge instances 20 +Running On-Demand m5n.4xlarge instances 20 +Running On-Demand m5n.8xlarge instances 20 +Running On-Demand m5n.large instances 20 +Running On-Demand m5n.metal instances 20 +Running On-Demand m5n.xlarge instances 20 +Running On-Demand p2.16xlarge instances 1 +Running On-Demand p2.8xlarge instances 1 +Running On-Demand p2.xlarge instances 1 +Running On-Demand p3.16xlarge instances 1 +Running On-Demand p3.2xlarge instances 1 +Running On-Demand p3.8xlarge instances 1 +Running On-Demand p3dn.24xlarge instances 1 +Running On-Demand r3.2xlarge instances |check| 20 +Running On-Demand r3.4xlarge instances 10 +Running On-Demand r3.8xlarge instances 5 +Running On-Demand r3.large instances 20 +Running On-Demand r3.xlarge instances 20 +Running On-Demand r4.16xlarge instances 1 +Running On-Demand r4.2xlarge instances 20 +Running On-Demand r4.4xlarge instances 10 +Running On-Demand r4.8xlarge instances 5 +Running On-Demand r4.large instances |check| 20 +Running On-Demand r4.xlarge instances |check| 20 +Running On-Demand r5.12xlarge instances 20 +Running On-Demand r5.16xlarge instances 20 +Running On-Demand r5.24xlarge instances 20 +Running On-Demand r5.2xlarge instances 20 +Running On-Demand r5.4xlarge instances 20 +Running On-Demand r5.8xlarge instances 20 +Running On-Demand r5.large instances 20 +Running On-Demand r5.metal instances 20 +Running On-Demand r5.xlarge instances |check| 20 +Running On-Demand r5a.12xlarge instances 20 +Running On-Demand r5a.16xlarge instances 20 +Running On-Demand r5a.24xlarge instances 20 +Running On-Demand r5a.2xlarge instances 20 +Running On-Demand r5a.4xlarge instances 20 +Running On-Demand r5a.8xlarge instances 20 +Running On-Demand r5a.large instances 20 +Running On-Demand r5a.xlarge instances 20 +Running On-Demand r5ad.12xlarge instances 20 +Running On-Demand r5ad.16xlarge instances 20 +Running On-Demand r5ad.24xlarge instances 20 +Running On-Demand r5ad.2xlarge instances 20 +Running On-Demand r5ad.4xlarge instances 20 +Running On-Demand r5ad.8xlarge instances 20 +Running On-Demand r5ad.large instances 20 +Running On-Demand r5ad.xlarge instances 20 +Running On-Demand r5d.12xlarge instances 20 +Running On-Demand r5d.16xlarge instances 20 +Running On-Demand r5d.24xlarge instances 20 +Running On-Demand r5d.2xlarge instances 20 +Running On-Demand r5d.4xlarge instances 20 +Running On-Demand r5d.8xlarge instances 20 +Running On-Demand r5d.large instances 20 +Running On-Demand r5d.metal instances 20 +Running On-Demand r5d.xlarge instances 20 +Running On-Demand r5dn.12xlarge instances 20 +Running On-Demand r5dn.16xlarge instances 20 +Running On-Demand r5dn.24xlarge instances 20 +Running On-Demand r5dn.2xlarge instances 20 +Running On-Demand r5dn.4xlarge instances 20 +Running On-Demand r5dn.8xlarge instances 20 +Running On-Demand r5dn.large instances 20 +Running On-Demand r5dn.metal instances 20 +Running On-Demand r5dn.xlarge instances 20 +Running On-Demand r5n.12xlarge instances 20 +Running On-Demand r5n.16xlarge instances 20 +Running On-Demand r5n.24xlarge instances 20 +Running On-Demand r5n.2xlarge instances 20 +Running On-Demand r5n.4xlarge instances 20 +Running On-Demand r5n.8xlarge instances 20 +Running On-Demand r5n.large instances 20 +Running On-Demand r5n.metal instances 20 +Running On-Demand r5n.xlarge instances 20 +Running On-Demand t1.micro instances |check| 20 +Running On-Demand t2.2xlarge instances 20 +Running On-Demand t2.large instances |check| 20 +Running On-Demand t2.medium instances |check| 20 +Running On-Demand t2.micro instances |check| 20 +Running On-Demand t2.nano instances |check| 20 +Running On-Demand t2.small instances |check| 20 +Running On-Demand t2.xlarge instances |check| 20 +Running On-Demand t3.2xlarge instances 20 +Running On-Demand t3.large instances |check| 20 +Running On-Demand t3.medium instances |check| 20 +Running On-Demand t3.micro instances 20 +Running On-Demand t3.nano instances 20 +Running On-Demand t3.small instances |check| 20 +Running On-Demand t3.xlarge instances 20 +Running On-Demand t3a.2xlarge instances 20 +Running On-Demand t3a.large instances 20 +Running On-Demand t3a.medium instances 20 +Running On-Demand t3a.micro instances 20 +Running On-Demand t3a.nano instances 20 +Running On-Demand t3a.small instances 20 +Running On-Demand t3a.xlarge instances 20 +Running On-Demand u-18tb1.metal instances 20 +Running On-Demand u-24tb1.metal instances 20 +Running On-Demand x1.16xlarge instances 20 +Running On-Demand x1.32xlarge instances 20 +Running On-Demand x1e.16xlarge instances 20 +Running On-Demand x1e.2xlarge instances 20 +Running On-Demand x1e.32xlarge instances 20 +Running On-Demand x1e.4xlarge instances 20 +Running On-Demand x1e.8xlarge instances 20 +Running On-Demand x1e.xlarge instances 20 +Running On-Demand z1d.12xlarge instances 20 +Running On-Demand z1d.2xlarge instances 20 +Running On-Demand z1d.3xlarge instances 20 +Running On-Demand z1d.6xlarge instances 20 +Running On-Demand z1d.large instances 20 +Running On-Demand z1d.xlarge instances 20 +Security groups per VPC 500 +VPC Elastic IP addresses (EIPs) |check| |check| 5 +VPC security groups per elastic network interface |check| 5 +================================================= =============== ======= ==== \ No newline at end of file diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index a9ea82a7..36f35bdd 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -23,6 +23,7 @@ What It Does an optional maximum time limit). See :ref:`Getting Started - Trusted Advisor ` for more information. +- Supports retrieving current limits from the Service Quotas service. See :ref:`getting_started.service_quotas` for more information. - Optionally send current usage and limit metrics to a :ref:`metrics store ` such as Datadog. - Optionally send warning/critical alerts to an :ref:`alert provider `, such as PagerDuty. @@ -61,7 +62,7 @@ Requirements **Either Docker in order to run via the** :ref:`docker image `, **or:** -* Python 2.7 or 3.4+. Python 2.6 and 3.3 are no longer supported. +* Python 3.5 or newer. Python 2.7 will not be supported as of January 1, 2010. * Python `VirtualEnv `_ and ``pip`` (recommended installation method; your OS/distribution should have packages for these) * `boto3 `_ >= 1.4.6 and its dependency `botocore `_ >= 1.6.0. @@ -205,6 +206,18 @@ IAM permission. See :ref:`Internals - Trusted Advisor ` for technical information on the implementation of Trusted Advisor polling. +.. _getting_started.service_quotas: + +Service Quotas service +---------------------- + +AWS' new `Service Quotas service `_ +provides a unified interface to retrieve current limits from many AWS services. These limit values are +second only to the services' own APIs (for the services that provide limit information via API), and are +much more current and complete than the information provided by Trusted Advisor. The introduction of +Service Quotas should greatly reduce the number of limits that need to be retrieved from Trusted Advisor +or specified manually. + .. _getting_started.permissions: Required Permissions diff --git a/docs/source/iam_policy.rst b/docs/source/iam_policy.rst index e08749e7..8d1795cc 100644 --- a/docs/source/iam_policy.rst +++ b/docs/source/iam_policy.rst @@ -31,85 +31,86 @@ services that do not affect the results of this program. "Statement": [ { "Action": [ - "apigateway:GET", - "apigateway:HEAD", - "apigateway:OPTIONS", - "autoscaling:DescribeAccountLimits", - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:DescribeLaunchConfigurations", - "cloudformation:DescribeAccountLimits", - "cloudformation:DescribeStacks", - "cloudtrail:DescribeTrails", - "cloudtrail:GetEventSelectors", - "ds:GetDirectoryLimits", - "dynamodb:DescribeLimits", - "dynamodb:DescribeTable", - "dynamodb:ListTables", - "ec2:DescribeAccountAttributes", - "ec2:DescribeAddresses", - "ec2:DescribeInstances", - "ec2:DescribeInternetGateways", - "ec2:DescribeNatGateways", - "ec2:DescribeNetworkAcls", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeReservedInstances", - "ec2:DescribeRouteTables", - "ec2:DescribeSecurityGroups", - "ec2:DescribeSnapshots", - "ec2:DescribeSpotDatafeedSubscription", - "ec2:DescribeSpotFleetInstances", - "ec2:DescribeSpotFleetRequestHistory", - "ec2:DescribeSpotFleetRequests", - "ec2:DescribeSpotInstanceRequests", - "ec2:DescribeSpotPriceHistory", - "ec2:DescribeSubnets", - "ec2:DescribeVolumes", - "ec2:DescribeVpcs", - "ec2:DescribeVpnGateways", - "ecs:DescribeClusters", - "ecs:DescribeServices", - "ecs:ListClusters", - "ecs:ListServices", - "elasticache:DescribeCacheClusters", - "elasticache:DescribeCacheParameterGroups", - "elasticache:DescribeCacheSecurityGroups", - "elasticache:DescribeCacheSubnetGroups", - "elasticbeanstalk:DescribeApplicationVersions", - "elasticbeanstalk:DescribeApplications", - "elasticbeanstalk:DescribeEnvironments", - "elasticfilesystem:DescribeFileSystems", - "elasticloadbalancing:DescribeAccountLimits", - "elasticloadbalancing:DescribeListeners", - "elasticloadbalancing:DescribeLoadBalancers", - "elasticloadbalancing:DescribeRules", - "elasticloadbalancing:DescribeTargetGroups", - "firehose:ListDeliveryStreams", - "iam:GetAccountSummary", - "lambda:GetAccountSettings", - "rds:DescribeAccountAttributes", - "rds:DescribeDBInstances", - "rds:DescribeDBParameterGroups", - "rds:DescribeDBSecurityGroups", - "rds:DescribeDBSnapshots", - "rds:DescribeDBSubnetGroups", - "rds:DescribeEventSubscriptions", - "rds:DescribeOptionGroups", - "rds:DescribeReservedDBInstances", - "redshift:DescribeClusterSnapshots", - "redshift:DescribeClusterSubnetGroups", - "route53:GetHostedZone", - "route53:GetHostedZoneLimit", - "route53:ListHostedZones", - "s3:ListAllMyBuckets", - "ses:GetSendQuota", - "support:*", - "trustedadvisor:Describe*", + "apigateway:GET", + "apigateway:HEAD", + "apigateway:OPTIONS", + "autoscaling:DescribeAccountLimits", + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "cloudformation:DescribeAccountLimits", + "cloudformation:DescribeStacks", + "cloudtrail:DescribeTrails", + "cloudtrail:GetEventSelectors", + "ds:GetDirectoryLimits", + "dynamodb:DescribeLimits", + "dynamodb:DescribeTable", + "dynamodb:ListTables", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeNatGateways", + "ec2:DescribeNetworkAcls", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeReservedInstances", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSnapshots", + "ec2:DescribeSpotDatafeedSubscription", + "ec2:DescribeSpotFleetInstances", + "ec2:DescribeSpotFleetRequestHistory", + "ec2:DescribeSpotFleetRequests", + "ec2:DescribeSpotInstanceRequests", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:DescribeVpcs", + "ec2:DescribeVpnGateways", + "ecs:DescribeClusters", + "ecs:DescribeServices", + "ecs:ListClusters", + "ecs:ListServices", + "elasticache:DescribeCacheClusters", + "elasticache:DescribeCacheParameterGroups", + "elasticache:DescribeCacheSecurityGroups", + "elasticache:DescribeCacheSubnetGroups", + "elasticbeanstalk:DescribeApplicationVersions", + "elasticbeanstalk:DescribeApplications", + "elasticbeanstalk:DescribeEnvironments", + "elasticfilesystem:DescribeFileSystems", + "elasticloadbalancing:DescribeAccountLimits", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "firehose:ListDeliveryStreams", + "iam:GetAccountSummary", + "lambda:GetAccountSettings", + "rds:DescribeAccountAttributes", + "rds:DescribeDBInstances", + "rds:DescribeDBParameterGroups", + "rds:DescribeDBSecurityGroups", + "rds:DescribeDBSnapshots", + "rds:DescribeDBSubnetGroups", + "rds:DescribeEventSubscriptions", + "rds:DescribeOptionGroups", + "rds:DescribeReservedDBInstances", + "redshift:DescribeClusterSnapshots", + "redshift:DescribeClusterSubnetGroups", + "route53:GetHostedZone", + "route53:GetHostedZoneLimit", + "route53:ListHostedZones", + "s3:ListAllMyBuckets", + "servicequotas:ListServiceQuotas", + "ses:GetSendQuota", + "support:*", + "trustedadvisor:Describe*", "trustedadvisor:RefreshCheck" - ], - "Effect": "Allow", + ], + "Effect": "Allow", "Resource": "*" } - ], + ], "Version": "2012-10-17" } diff --git a/docs/source/internals.rst b/docs/source/internals.rst index 21df9304..21b909bc 100644 --- a/docs/source/internals.rst +++ b/docs/source/internals.rst @@ -38,7 +38,7 @@ to the services and limits without any connection to AWS. This is utilized by th .. _internals.trusted_advisor: Trusted Advisor ------------------ +--------------- When :py:class:`~awslimitchecker.checker.AwsLimitChecker` is initialized, it also initializes an instance of :py:class:`~awslimitchecker.trustedadvisor.TrustedAdvisor`. In :py:meth:`~.AwsLimitChecker.get_limits`, @@ -112,6 +112,21 @@ For use via Python, these same parameters (``ta_refresh_mode`` and ``ta_refresh_ are exposed as parameters on the :py:class:`~awslimitchecker.checker.AwsLimitChecker` constructor. +.. _internals.quotas: + +Service Quotas service +---------------------- + +Unless use of Serivce Quotas is disabled with the ``--skip-quotas`` command line option or by passing ``skip_quotas=False`` to the :py:class:`~awslimitchecker.checker.AwsLimitChecker` constructor, awslimitchecker will retrieve all relevant data from the Service Quotas service. In the :py:class:`~.AwsLimitChecker` constructor (so long as ``skip_quotas`` is True), an instance of the :py:class:`~.ServiceQuotasClient` class is constructed, passing in our boto3 connection keyword arguments for the current region. This client class instance is then passed to the constructor of every Service class (:py:class:`~._AwsService` subclass) when the class is created, via the ``quotas_client`` argument. Each :py:class:`~._AwsService` class stores this as the ``_quotas_client`` instance variable. + +As the :py:class:`~.AwsLimitChecker` class iterates over all (configured) services in its :py:meth:`~.AwsLimitChecker.get_limits`, :py:meth:`~.AwsLimitChecker.find_usage`, and :py:meth:`~.AwsLimitChecker.check_thresholds` methods, it will call the service class's :py:meth:`~._AwsService._update_service_quotas` method after calling :py:meth:`~.TrustedAdvisor.update_limits` and the service class's ``_update_limits_from_api()`` method (if present), and before the actual operation of getting limits, finding usage, or checking thresholds. + +The :py:meth:`._AwsService._update_service_quotas` method will iterate through all limits (:py:class:`~.AwsLimit`) for the service and call the :py:meth:`~.ServiceQuotasClient.get_quota_value` method for each. Assuming it returns a non-``None`` result, that result will be passed to the limit's :py:meth:`~.AwsLimit._set_quotas_limit` method for later use in :py:meth:`~.AwsLimit.get_limit`. + +When retrieving values from Service Quotas, the ``ServiceCode`` is taken from the :py:attr:`._AwsService.quotas_service_code` attribute on the Service class. If that is set to ``None``, Service Quotas will not be consulted for that service. The ``ServiceCode`` can also be overridden on a per-limit basis via the ``quotas_service_code`` argument to the :py:class:`~.AwsLimit` constructor. The ``QuotaName`` used by each limit defaults to the limit name itself (:py:class:`.AwsLimit` instance variable ``name``) but can be overridden with the ``quota_name`` argument to the :py:class:`~.AwsLimit` constructor. + +Note that quota names are stored and compared in lower case. + Service API Limit Information ----------------------------- @@ -129,8 +144,9 @@ The value used for a limit is the first match in the following list: 1. Limit Override (set at runtime) 2. API Limit -3. Trusted Advisor -4. Hard-coded default +3. Service Quotas +4. Trusted Advisor +5. Hard-coded default Threshold Overrides ------------------- diff --git a/docs/source/limits.rst b/docs/source/limits.rst index 34ce94ea..72814682 100644 --- a/docs/source/limits.rst +++ b/docs/source/limits.rst @@ -21,121 +21,179 @@ with a |check| were detected as being returned by Trusted Advisor as of the last release. Note that not all accounts can access Trusted Advisor, or can access all limits known by Trusted Advisor. +**Limits with a** |check| **in the "Quotas" column can be retrieved from +the Service Quotas service**; this information is supposed to be accurate +and up-to-date, but is likely less accurate than the service's own API. +Limits retrieved from Service Quotas take precedence over Trusted Advisor +and default limits. + **Limits with a** |check| **in the "API" column can be retrieved directly from the corresponding Service API**; this information should be the most accurate and up-to-date, as it is retrieved directly from the service that evaluates and enforces limits. Limits retrieved via service API take precedence over -Trusted Advisor and default limits. +Trusted Advisor, Service Quotas, and default limits. .. _limits.ApiGateway: ApiGateway ----------- -=============================== =============== ======= ==== -Limit Trusted Advisor API Default -=============================== =============== ======= ==== -API keys per account 500 -Client certificates per account 60 -Custom authorizers per API 10 -Documentation parts per API 2000 -Edge APIs per account 120 -Private APIs per account 600 -Regional APIs per account 600 -Resources per API 300 -Stages per API 10 -Usage plans per account 300 -VPC Links per account 5 -=============================== =============== ======= ==== +=============================== =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +=============================== =============== ======== ======= ==== +API keys per account |check| 500 +Client certificates per account |check| 60 +Custom authorizers per API 10 +Documentation parts per API 2000 +Edge APIs per account |check| 120 +Private APIs per account |check| 600 +Regional APIs per account |check| 600 +Resources per API |check| 300 +Stages per API |check| 10 +Usage plans per account |check| 300 +VPC Links per account |check| 5 +=============================== =============== ======== ======= ==== .. _limits.AutoScaling: AutoScaling ------------ -===================== =============== ======= === -Limit Trusted Advisor API Default -===================== =============== ======= === -Auto Scaling groups |check| |check| 200 -Launch configurations |check| |check| 200 -===================== =============== ======= === +===================== =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +===================== =============== ======== ======= === +Auto Scaling groups |check| |check| |check| 200 +Launch configurations |check| |check| |check| 200 +===================== =============== ======== ======= === .. _limits.CloudFormation: CloudFormation --------------- -====== =============== ======= === -Limit Trusted Advisor API Default -====== =============== ======= === -Stacks |check| |check| 200 -====== =============== ======= === +====== =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +====== =============== ======== ======= === +Stacks |check| |check| |check| 200 +====== =============== ======== ======= === .. _limits.CloudTrail: CloudTrail ----------- -========================= =============== ======= === -Limit Trusted Advisor API Default -========================= =============== ======= === -Data Resources Per Trail 250 -Event Selectors Per Trail 5 -Trails Per Region 5 -========================= =============== ======= === +========================= =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +========================= =============== ======== ======= === +Data Resources Per Trail 250 +Event Selectors Per Trail 5 +Trails Per Region 5 +========================= =============== ======== ======= === .. _limits.Directory Service: Directory Service ------------------ -==================== =============== ======= == -Limit Trusted Advisor API Default -==================== =============== ======= == -CloudOnlyDirectories |check| 10 -CloudOnlyMicrosoftAD |check| 10 -ConnectedDirectories |check| 10 -==================== =============== ======= == +==================== =============== ======== ======= == +Limit Trusted Advisor Quotas API Default +==================== =============== ======== ======= == +CloudOnlyDirectories |check| 10 +CloudOnlyMicrosoftAD |check| 10 +ConnectedDirectories |check| 10 +==================== =============== ======== ======= == .. _limits.DynamoDB: DynamoDB --------- -================================ =============== ======= ===== -Limit Trusted Advisor API Default -================================ =============== ======= ===== -Account Max Read Capacity Units |check| 80000 -Account Max Write Capacity Units |check| 80000 -Global Secondary Indexes 20 -Local Secondary Indexes 5 -Table Max Read Capacity Units |check| 40000 -Table Max Write Capacity Units |check| 40000 -Tables Per Region 256 -================================ =============== ======= ===== +================================ =============== ======== ======= ===== +Limit Trusted Advisor Quotas API Default +================================ =============== ======== ======= ===== +Account Max Read Capacity Units |check| |check| 80000 +Account Max Write Capacity Units |check| |check| 80000 +Global Secondary Indexes 20 +Local Secondary Indexes 5 +Table Max Read Capacity Units |check| |check| 40000 +Table Max Write Capacity Units |check| |check| 40000 +Tables Per Region 256 +================================ =============== ======== ======= ===== .. _limits.EBS: EBS ---- -=============================================== =============== ======= ====== -Limit Trusted Advisor API Default -=============================================== =============== ======= ====== -Active snapshots |check| 10000 -Active volumes |check| 5000 -Cold (HDD) volume storage (GiB) 307200 -General Purpose (SSD) volume storage (GiB) |check| 102400 -Magnetic volume storage (GiB) |check| 20480 -Provisioned IOPS |check| 200000 -Provisioned IOPS (SSD) storage (GiB) |check| 102400 -Throughput Optimized (HDD) volume storage (GiB) 307200 -=============================================== =============== ======= ====== +=============================================== =============== ======== ======= ====== +Limit Trusted Advisor Quotas API Default +=============================================== =============== ======== ======= ====== +Active snapshots |check| |check| 10000 +Active volumes 5000 +Cold (HDD) volume storage (GiB) |check| 307200 +General Purpose (SSD) volume storage (GiB) |check| |check| 307200 +Magnetic volume storage (GiB) |check| |check| 307200 +Provisioned IOPS |check| |check| 200000 +Provisioned IOPS (SSD) storage (GiB) |check| |check| 307200 +Throughput Optimized (HDD) volume storage (GiB) |check| 307200 +=============================================== =============== ======== ======= ====== .. _limits.EC2: EC2 ----- +--- + + +As of October 2019, the "standard" EC2 regions use the new +`vCPU-based limits `__, while the China (``cn-``) and GovCloud (``us-gov-``) +regions still use the old per-instance-type limits. Please see the sections +for either :ref:`limits.ec2-standard` or :ref:`limits.ec2-nonvcpu` for +details. + +.. _limits.ec2-standard: + +EC2 - Standard Regions +---------------------- + + +**Note on On-Demand vs Reserved Instances:** The EC2 limits for +"Running On-Demand" EC2 Instances apply only to On-Demand instances, +not Reserved Instances. If you list all EC2 instances that are +running in the Console or API, you'll get back instances of all types +(On-Demand, Reserved, etc.). The value that awslimitchecker reports +for Running On-Demand Instances current usage will *not* match the +number of instances you see in the Console or API. + +**Important:** The limits for **Running On-Demand Instances** are now +measured in vCPU count per instance family, not instance count per instance +type. + + +==================================================================== =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +==================================================================== =============== ======== ======= ==== +Elastic IP addresses (EIPs) |check| |check| |check| 5 +Max active spot fleets per region 1000 +Max launch specifications per spot fleet 50 +Max spot instance requests per region 20 +Max target capacity for all spot fleets in region 5000 +Max target capacity per spot fleet 3000 +Rules per VPC security group 50 +Running On-Demand All F instances |check| 128 +Running On-Demand All G instances |check| 128 +Running On-Demand All P instances |check| 128 +Running On-Demand All Standard (A, C, D, H, I, M, R, T, Z) instances |check| 1152 +Running On-Demand All X instances |check| 128 +Security groups per VPC 500 +VPC Elastic IP addresses (EIPs) |check| |check| |check| 5 +VPC security groups per elastic network interface |check| 5 +==================================================================== =============== ======== ======= ==== + +.. _limits.ec2-nonvcpu: + +EC2 - China and GovCloud +------------------------ **Note on On-Demand vs Reserved Instances:** The EC2 limits for @@ -150,192 +208,285 @@ number of instances you see in the Console or API. ================================================= =============== ======= ==== Limit Trusted Advisor API Default ================================================= =============== ======= ==== -Elastic IP addresses (EIPs) |check| |check| 5 +Elastic IP addresses (EIPs) |check| |check| 5 Max active spot fleets per region 1000 -Max launch specifications per spot fleet 50 -Max spot instance requests per region 20 +Max launch specifications per spot fleet 50 +Max spot instance requests per region 20 Max target capacity for all spot fleets in region 5000 Max target capacity per spot fleet 3000 -Rules per VPC security group 50 -Running On-Demand EC2 instances |check| 20 -Running On-Demand a1.2xlarge instances 20 -Running On-Demand a1.4xlarge instances 20 -Running On-Demand a1.large instances 20 -Running On-Demand a1.medium instances 20 -Running On-Demand a1.xlarge instances 20 -Running On-Demand c1.medium instances 20 -Running On-Demand c1.xlarge instances 20 -Running On-Demand c3.2xlarge instances 20 -Running On-Demand c3.4xlarge instances 20 -Running On-Demand c3.8xlarge instances 20 -Running On-Demand c3.large instances |check| 20 -Running On-Demand c3.xlarge instances |check| 20 -Running On-Demand c4.2xlarge instances |check| 20 -Running On-Demand c4.4xlarge instances |check| 10 -Running On-Demand c4.8xlarge instances 5 -Running On-Demand c4.large instances |check| 20 -Running On-Demand c4.xlarge instances |check| 20 -Running On-Demand c5.18xlarge instances 5 -Running On-Demand c5.2xlarge instances |check| 20 -Running On-Demand c5.4xlarge instances |check| 10 -Running On-Demand c5.9xlarge instances 5 -Running On-Demand c5.large instances |check| 20 -Running On-Demand c5.xlarge instances 20 -Running On-Demand c5d.18xlarge instances 20 -Running On-Demand c5d.2xlarge instances 20 -Running On-Demand c5d.4xlarge instances 20 -Running On-Demand c5d.9xlarge instances 20 -Running On-Demand c5d.large instances 20 -Running On-Demand c5d.xlarge instances 20 -Running On-Demand c5n.18xlarge instances 20 -Running On-Demand c5n.2xlarge instances 20 -Running On-Demand c5n.4xlarge instances 20 -Running On-Demand c5n.9xlarge instances 20 -Running On-Demand c5n.large instances 20 -Running On-Demand c5n.xlarge instances 20 -Running On-Demand cc1.4xlarge instances 20 -Running On-Demand cc2.8xlarge instances 20 -Running On-Demand cg1.4xlarge instances 2 -Running On-Demand cr1.8xlarge instances 2 -Running On-Demand d2.2xlarge instances 20 -Running On-Demand d2.4xlarge instances 10 -Running On-Demand d2.8xlarge instances 5 -Running On-Demand d2.xlarge instances 20 -Running On-Demand f1.16xlarge instances 20 -Running On-Demand f1.2xlarge instances 20 -Running On-Demand f1.4xlarge instances 20 -Running On-Demand g2.2xlarge instances 5 -Running On-Demand g2.8xlarge instances 2 -Running On-Demand g3.16xlarge instances 1 -Running On-Demand g3.4xlarge instances 1 -Running On-Demand g3.8xlarge instances 1 -Running On-Demand g3s.xlarge instances 20 -Running On-Demand h1.16xlarge instances 5 -Running On-Demand h1.2xlarge instances 20 -Running On-Demand h1.4xlarge instances 20 -Running On-Demand h1.8xlarge instances 10 -Running On-Demand hi1.4xlarge instances 2 -Running On-Demand hs1.8xlarge instances 2 -Running On-Demand i2.2xlarge instances 8 -Running On-Demand i2.4xlarge instances 4 -Running On-Demand i2.8xlarge instances 2 -Running On-Demand i2.xlarge instances 8 -Running On-Demand i3.16xlarge instances 2 -Running On-Demand i3.2xlarge instances 2 -Running On-Demand i3.4xlarge instances 2 -Running On-Demand i3.8xlarge instances 2 -Running On-Demand i3.large instances 2 -Running On-Demand i3.metal instances 20 -Running On-Demand i3.xlarge instances 2 -Running On-Demand m1.large instances 20 -Running On-Demand m1.medium instances 20 -Running On-Demand m1.small instances |check| 20 -Running On-Demand m1.xlarge instances 20 -Running On-Demand m2.2xlarge instances 20 -Running On-Demand m2.4xlarge instances 20 -Running On-Demand m2.xlarge instances 20 -Running On-Demand m3.2xlarge instances |check| 20 -Running On-Demand m3.large instances |check| 20 -Running On-Demand m3.medium instances |check| 20 -Running On-Demand m3.xlarge instances |check| 20 -Running On-Demand m4.10xlarge instances 5 -Running On-Demand m4.16xlarge instances 5 -Running On-Demand m4.2xlarge instances |check| 20 -Running On-Demand m4.4xlarge instances |check| 10 -Running On-Demand m4.large instances |check| 20 -Running On-Demand m4.xlarge instances |check| 20 -Running On-Demand m5.12xlarge instances 5 -Running On-Demand m5.24xlarge instances 5 -Running On-Demand m5.2xlarge instances 20 -Running On-Demand m5.4xlarge instances 10 -Running On-Demand m5.large instances |check| 20 -Running On-Demand m5.xlarge instances 20 -Running On-Demand m5a.12xlarge instances 20 -Running On-Demand m5a.24xlarge instances 20 -Running On-Demand m5a.2xlarge instances 20 -Running On-Demand m5a.4xlarge instances 20 -Running On-Demand m5a.large instances 20 -Running On-Demand m5a.xlarge instances 20 -Running On-Demand m5d.12xlarge instances 20 -Running On-Demand m5d.24xlarge instances 20 -Running On-Demand m5d.2xlarge instances 20 -Running On-Demand m5d.4xlarge instances 20 -Running On-Demand m5d.large instances 20 -Running On-Demand m5d.xlarge instances 20 -Running On-Demand p2.16xlarge instances 1 -Running On-Demand p2.8xlarge instances 1 -Running On-Demand p2.xlarge instances 1 -Running On-Demand p3.16xlarge instances 1 -Running On-Demand p3.2xlarge instances 1 -Running On-Demand p3.8xlarge instances 1 -Running On-Demand p3dn.24xlarge instances 1 -Running On-Demand r3.2xlarge instances |check| 20 -Running On-Demand r3.4xlarge instances |check| 10 -Running On-Demand r3.8xlarge instances 5 -Running On-Demand r3.large instances 20 -Running On-Demand r3.xlarge instances 20 -Running On-Demand r4.16xlarge instances 1 -Running On-Demand r4.2xlarge instances 20 -Running On-Demand r4.4xlarge instances |check| 10 -Running On-Demand r4.8xlarge instances 5 -Running On-Demand r4.large instances |check| 20 -Running On-Demand r4.xlarge instances |check| 20 -Running On-Demand r5.12xlarge instances 20 -Running On-Demand r5.16xlarge instances 20 -Running On-Demand r5.24xlarge instances 20 -Running On-Demand r5.2xlarge instances 20 -Running On-Demand r5.4xlarge instances 20 -Running On-Demand r5.8xlarge instances 20 -Running On-Demand r5.large instances |check| 20 -Running On-Demand r5.metal instances 20 -Running On-Demand r5.xlarge instances 20 -Running On-Demand r5a.12xlarge instances 20 -Running On-Demand r5a.24xlarge instances 20 -Running On-Demand r5a.2xlarge instances 20 -Running On-Demand r5a.4xlarge instances 20 -Running On-Demand r5a.large instances 20 -Running On-Demand r5a.xlarge instances 20 -Running On-Demand r5d.12xlarge instances 20 -Running On-Demand r5d.16xlarge instances 20 -Running On-Demand r5d.24xlarge instances 20 -Running On-Demand r5d.2xlarge instances 20 -Running On-Demand r5d.4xlarge instances 20 -Running On-Demand r5d.8xlarge instances 20 -Running On-Demand r5d.large instances 20 -Running On-Demand r5d.metal instances 20 -Running On-Demand r5d.xlarge instances 20 -Running On-Demand t1.micro instances |check| 20 -Running On-Demand t2.2xlarge instances 20 -Running On-Demand t2.large instances |check| 20 -Running On-Demand t2.medium instances |check| 20 -Running On-Demand t2.micro instances |check| 20 -Running On-Demand t2.nano instances |check| 20 -Running On-Demand t2.small instances |check| 20 -Running On-Demand t2.xlarge instances |check| 20 -Running On-Demand t3.2xlarge instances 20 -Running On-Demand t3.large instances |check| 20 -Running On-Demand t3.medium instances |check| 20 -Running On-Demand t3.micro instances |check| 20 -Running On-Demand t3.nano instances 20 -Running On-Demand t3.small instances |check| 20 -Running On-Demand t3.xlarge instances 20 -Running On-Demand x1.16xlarge instances 20 -Running On-Demand x1.32xlarge instances 20 -Running On-Demand x1e.16xlarge instances 20 -Running On-Demand x1e.2xlarge instances 20 -Running On-Demand x1e.32xlarge instances 20 -Running On-Demand x1e.4xlarge instances 20 -Running On-Demand x1e.8xlarge instances 20 -Running On-Demand x1e.xlarge instances 20 -Running On-Demand z1d.12xlarge instances 20 -Running On-Demand z1d.2xlarge instances 20 -Running On-Demand z1d.3xlarge instances 20 -Running On-Demand z1d.6xlarge instances 20 -Running On-Demand z1d.large instances 20 -Running On-Demand z1d.xlarge instances 20 -Security groups per VPC 500 -VPC Elastic IP addresses (EIPs) |check| |check| 5 -VPC security groups per elastic network interface |check| 5 +Rules per VPC security group 50 +Running On-Demand EC2 instances |check| 20 +Running On-Demand a1.2xlarge instances 20 +Running On-Demand a1.4xlarge instances 20 +Running On-Demand a1.large instances 20 +Running On-Demand a1.medium instances 20 +Running On-Demand a1.metal instances 20 +Running On-Demand a1.xlarge instances 20 +Running On-Demand c1.medium instances 20 +Running On-Demand c1.xlarge instances 20 +Running On-Demand c3.2xlarge instances 20 +Running On-Demand c3.4xlarge instances 20 +Running On-Demand c3.8xlarge instances 20 +Running On-Demand c3.large instances |check| 20 +Running On-Demand c3.xlarge instances |check| 20 +Running On-Demand c4.2xlarge instances |check| 20 +Running On-Demand c4.4xlarge instances |check| 10 +Running On-Demand c4.8xlarge instances |check| 5 +Running On-Demand c4.large instances |check| 20 +Running On-Demand c4.xlarge instances |check| 20 +Running On-Demand c5.12xlarge instances 20 +Running On-Demand c5.18xlarge instances 5 +Running On-Demand c5.24xlarge instances 20 +Running On-Demand c5.2xlarge instances |check| 20 +Running On-Demand c5.4xlarge instances |check| 10 +Running On-Demand c5.9xlarge instances 5 +Running On-Demand c5.large instances |check| 20 +Running On-Demand c5.metal instances 20 +Running On-Demand c5.xlarge instances 20 +Running On-Demand c5d.12xlarge instances 20 +Running On-Demand c5d.18xlarge instances 20 +Running On-Demand c5d.24xlarge instances 20 +Running On-Demand c5d.2xlarge instances 20 +Running On-Demand c5d.4xlarge instances 20 +Running On-Demand c5d.9xlarge instances 20 +Running On-Demand c5d.large instances 20 +Running On-Demand c5d.metal instances 20 +Running On-Demand c5d.xlarge instances 20 +Running On-Demand c5n.18xlarge instances 20 +Running On-Demand c5n.2xlarge instances 20 +Running On-Demand c5n.4xlarge instances 20 +Running On-Demand c5n.9xlarge instances 20 +Running On-Demand c5n.large instances 20 +Running On-Demand c5n.metal instances 20 +Running On-Demand c5n.xlarge instances 20 +Running On-Demand cc1.4xlarge instances 20 +Running On-Demand cc2.8xlarge instances 20 +Running On-Demand cg1.4xlarge instances 2 +Running On-Demand cr1.8xlarge instances 2 +Running On-Demand d2.2xlarge instances 20 +Running On-Demand d2.4xlarge instances 10 +Running On-Demand d2.8xlarge instances 5 +Running On-Demand d2.xlarge instances 20 +Running On-Demand f1.16xlarge instances 20 +Running On-Demand f1.2xlarge instances 20 +Running On-Demand f1.4xlarge instances 20 +Running On-Demand g2.2xlarge instances 5 +Running On-Demand g2.8xlarge instances 2 +Running On-Demand g3.16xlarge instances 1 +Running On-Demand g3.4xlarge instances 1 +Running On-Demand g3.8xlarge instances 1 +Running On-Demand g3s.xlarge instances 20 +Running On-Demand g4dn.12xlarge instances 20 +Running On-Demand g4dn.16xlarge instances 20 +Running On-Demand g4dn.2xlarge instances 20 +Running On-Demand g4dn.4xlarge instances 20 +Running On-Demand g4dn.8xlarge instances 20 +Running On-Demand g4dn.metal instances 20 +Running On-Demand g4dn.xlarge instances 20 +Running On-Demand h1.16xlarge instances 5 +Running On-Demand h1.2xlarge instances 20 +Running On-Demand h1.4xlarge instances 20 +Running On-Demand h1.8xlarge instances 10 +Running On-Demand hi1.4xlarge instances 2 +Running On-Demand hs1.8xlarge instances 2 +Running On-Demand i2.2xlarge instances 8 +Running On-Demand i2.4xlarge instances 4 +Running On-Demand i2.8xlarge instances 2 +Running On-Demand i2.xlarge instances 8 +Running On-Demand i3.16xlarge instances 2 +Running On-Demand i3.2xlarge instances 2 +Running On-Demand i3.4xlarge instances 2 +Running On-Demand i3.8xlarge instances 2 +Running On-Demand i3.large instances 2 +Running On-Demand i3.metal instances 20 +Running On-Demand i3.xlarge instances 2 +Running On-Demand i3en.12xlarge instances 20 +Running On-Demand i3en.24xlarge instances 20 +Running On-Demand i3en.2xlarge instances 20 +Running On-Demand i3en.3xlarge instances 20 +Running On-Demand i3en.6xlarge instances 20 +Running On-Demand i3en.large instances 20 +Running On-Demand i3en.xlarge instances 20 +Running On-Demand m1.large instances 20 +Running On-Demand m1.medium instances 20 +Running On-Demand m1.small instances |check| 20 +Running On-Demand m1.xlarge instances 20 +Running On-Demand m2.2xlarge instances 20 +Running On-Demand m2.4xlarge instances 20 +Running On-Demand m2.xlarge instances 20 +Running On-Demand m3.2xlarge instances |check| 20 +Running On-Demand m3.large instances |check| 20 +Running On-Demand m3.medium instances |check| 20 +Running On-Demand m3.xlarge instances |check| 20 +Running On-Demand m4.10xlarge instances 5 +Running On-Demand m4.16xlarge instances |check| 5 +Running On-Demand m4.2xlarge instances |check| 20 +Running On-Demand m4.4xlarge instances |check| 10 +Running On-Demand m4.large instances |check| 20 +Running On-Demand m4.xlarge instances |check| 20 +Running On-Demand m5.12xlarge instances 5 +Running On-Demand m5.16xlarge instances 20 +Running On-Demand m5.24xlarge instances 5 +Running On-Demand m5.2xlarge instances 20 +Running On-Demand m5.4xlarge instances 10 +Running On-Demand m5.8xlarge instances 20 +Running On-Demand m5.large instances |check| 20 +Running On-Demand m5.metal instances 20 +Running On-Demand m5.xlarge instances 20 +Running On-Demand m5a.12xlarge instances 20 +Running On-Demand m5a.16xlarge instances 20 +Running On-Demand m5a.24xlarge instances 20 +Running On-Demand m5a.2xlarge instances 20 +Running On-Demand m5a.4xlarge instances 20 +Running On-Demand m5a.8xlarge instances 20 +Running On-Demand m5a.large instances 20 +Running On-Demand m5a.xlarge instances 20 +Running On-Demand m5ad.12xlarge instances 20 +Running On-Demand m5ad.16xlarge instances 20 +Running On-Demand m5ad.24xlarge instances 20 +Running On-Demand m5ad.2xlarge instances 20 +Running On-Demand m5ad.4xlarge instances 20 +Running On-Demand m5ad.8xlarge instances 20 +Running On-Demand m5ad.large instances 20 +Running On-Demand m5ad.xlarge instances 20 +Running On-Demand m5d.12xlarge instances 20 +Running On-Demand m5d.16xlarge instances 20 +Running On-Demand m5d.24xlarge instances 20 +Running On-Demand m5d.2xlarge instances 20 +Running On-Demand m5d.4xlarge instances 20 +Running On-Demand m5d.8xlarge instances 20 +Running On-Demand m5d.large instances 20 +Running On-Demand m5d.metal instances 20 +Running On-Demand m5d.xlarge instances 20 +Running On-Demand m5dn.12xlarge instances 20 +Running On-Demand m5dn.16xlarge instances 20 +Running On-Demand m5dn.24xlarge instances 20 +Running On-Demand m5dn.2xlarge instances 20 +Running On-Demand m5dn.4xlarge instances 20 +Running On-Demand m5dn.8xlarge instances 20 +Running On-Demand m5dn.large instances 20 +Running On-Demand m5dn.metal instances 20 +Running On-Demand m5dn.xlarge instances 20 +Running On-Demand m5n.12xlarge instances 20 +Running On-Demand m5n.16xlarge instances 20 +Running On-Demand m5n.24xlarge instances 20 +Running On-Demand m5n.2xlarge instances 20 +Running On-Demand m5n.4xlarge instances 20 +Running On-Demand m5n.8xlarge instances 20 +Running On-Demand m5n.large instances 20 +Running On-Demand m5n.metal instances 20 +Running On-Demand m5n.xlarge instances 20 +Running On-Demand p2.16xlarge instances 1 +Running On-Demand p2.8xlarge instances 1 +Running On-Demand p2.xlarge instances 1 +Running On-Demand p3.16xlarge instances 1 +Running On-Demand p3.2xlarge instances 1 +Running On-Demand p3.8xlarge instances 1 +Running On-Demand p3dn.24xlarge instances 1 +Running On-Demand r3.2xlarge instances |check| 20 +Running On-Demand r3.4xlarge instances 10 +Running On-Demand r3.8xlarge instances 5 +Running On-Demand r3.large instances 20 +Running On-Demand r3.xlarge instances 20 +Running On-Demand r4.16xlarge instances 1 +Running On-Demand r4.2xlarge instances 20 +Running On-Demand r4.4xlarge instances 10 +Running On-Demand r4.8xlarge instances 5 +Running On-Demand r4.large instances |check| 20 +Running On-Demand r4.xlarge instances |check| 20 +Running On-Demand r5.12xlarge instances 20 +Running On-Demand r5.16xlarge instances 20 +Running On-Demand r5.24xlarge instances 20 +Running On-Demand r5.2xlarge instances 20 +Running On-Demand r5.4xlarge instances 20 +Running On-Demand r5.8xlarge instances 20 +Running On-Demand r5.large instances 20 +Running On-Demand r5.metal instances 20 +Running On-Demand r5.xlarge instances |check| 20 +Running On-Demand r5a.12xlarge instances 20 +Running On-Demand r5a.16xlarge instances 20 +Running On-Demand r5a.24xlarge instances 20 +Running On-Demand r5a.2xlarge instances 20 +Running On-Demand r5a.4xlarge instances 20 +Running On-Demand r5a.8xlarge instances 20 +Running On-Demand r5a.large instances 20 +Running On-Demand r5a.xlarge instances 20 +Running On-Demand r5ad.12xlarge instances 20 +Running On-Demand r5ad.16xlarge instances 20 +Running On-Demand r5ad.24xlarge instances 20 +Running On-Demand r5ad.2xlarge instances 20 +Running On-Demand r5ad.4xlarge instances 20 +Running On-Demand r5ad.8xlarge instances 20 +Running On-Demand r5ad.large instances 20 +Running On-Demand r5ad.xlarge instances 20 +Running On-Demand r5d.12xlarge instances 20 +Running On-Demand r5d.16xlarge instances 20 +Running On-Demand r5d.24xlarge instances 20 +Running On-Demand r5d.2xlarge instances 20 +Running On-Demand r5d.4xlarge instances 20 +Running On-Demand r5d.8xlarge instances 20 +Running On-Demand r5d.large instances 20 +Running On-Demand r5d.metal instances 20 +Running On-Demand r5d.xlarge instances 20 +Running On-Demand r5dn.12xlarge instances 20 +Running On-Demand r5dn.16xlarge instances 20 +Running On-Demand r5dn.24xlarge instances 20 +Running On-Demand r5dn.2xlarge instances 20 +Running On-Demand r5dn.4xlarge instances 20 +Running On-Demand r5dn.8xlarge instances 20 +Running On-Demand r5dn.large instances 20 +Running On-Demand r5dn.metal instances 20 +Running On-Demand r5dn.xlarge instances 20 +Running On-Demand r5n.12xlarge instances 20 +Running On-Demand r5n.16xlarge instances 20 +Running On-Demand r5n.24xlarge instances 20 +Running On-Demand r5n.2xlarge instances 20 +Running On-Demand r5n.4xlarge instances 20 +Running On-Demand r5n.8xlarge instances 20 +Running On-Demand r5n.large instances 20 +Running On-Demand r5n.metal instances 20 +Running On-Demand r5n.xlarge instances 20 +Running On-Demand t1.micro instances |check| 20 +Running On-Demand t2.2xlarge instances 20 +Running On-Demand t2.large instances |check| 20 +Running On-Demand t2.medium instances |check| 20 +Running On-Demand t2.micro instances |check| 20 +Running On-Demand t2.nano instances |check| 20 +Running On-Demand t2.small instances |check| 20 +Running On-Demand t2.xlarge instances |check| 20 +Running On-Demand t3.2xlarge instances 20 +Running On-Demand t3.large instances |check| 20 +Running On-Demand t3.medium instances |check| 20 +Running On-Demand t3.micro instances 20 +Running On-Demand t3.nano instances 20 +Running On-Demand t3.small instances |check| 20 +Running On-Demand t3.xlarge instances 20 +Running On-Demand t3a.2xlarge instances 20 +Running On-Demand t3a.large instances 20 +Running On-Demand t3a.medium instances 20 +Running On-Demand t3a.micro instances 20 +Running On-Demand t3a.nano instances 20 +Running On-Demand t3a.small instances 20 +Running On-Demand t3a.xlarge instances 20 +Running On-Demand u-18tb1.metal instances 20 +Running On-Demand u-24tb1.metal instances 20 +Running On-Demand x1.16xlarge instances 20 +Running On-Demand x1.32xlarge instances 20 +Running On-Demand x1e.16xlarge instances 20 +Running On-Demand x1e.2xlarge instances 20 +Running On-Demand x1e.32xlarge instances 20 +Running On-Demand x1e.4xlarge instances 20 +Running On-Demand x1e.8xlarge instances 20 +Running On-Demand x1e.xlarge instances 20 +Running On-Demand z1d.12xlarge instances 20 +Running On-Demand z1d.2xlarge instances 20 +Running On-Demand z1d.3xlarge instances 20 +Running On-Demand z1d.6xlarge instances 20 +Running On-Demand z1d.large instances 20 +Running On-Demand z1d.xlarge instances 20 +Security groups per VPC 500 +VPC Elastic IP addresses (EIPs) |check| |check| 5 +VPC security groups per elastic network interface |check| 5 ================================================= =============== ======= ==== .. _limits.ECS: @@ -343,156 +494,156 @@ VPC security groups per elastic network interface |check| 5 ECS ---- -===================================== =============== ======= ==== -Limit Trusted Advisor API Default -===================================== =============== ======= ==== -Clusters 2000 -Container Instances per Cluster 2000 -EC2 Tasks per Service (desired count) 1000 -Fargate Tasks 50 -Services per Cluster 1000 -===================================== =============== ======= ==== +===================================== =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +===================================== =============== ======== ======= ==== +Clusters 2000 +Container Instances per Cluster 2000 +EC2 Tasks per Service (desired count) 1000 +Fargate Tasks 50 +Services per Cluster 1000 +===================================== =============== ======== ======= ==== .. _limits.EFS: EFS ---- -============ =============== ======= == -Limit Trusted Advisor API Default -============ =============== ======= == -File systems 70 -============ =============== ======= == +============ =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +============ =============== ======== ======= ==== +File systems |check| 1000 +============ =============== ======== ======= ==== .. _limits.ELB: ELB ---- -========================================== =============== ======= ==== -Limit Trusted Advisor API Default -========================================== =============== ======= ==== -Application load balancers |check| 20 -Certificates per application load balancer 25 -Classic load balancers |check| 20 -Listeners per application load balancer |check| 50 -Listeners per load balancer |check| 100 -Listeners per network load balancer |check| 50 -Network load balancers |check| 20 -Registered instances per load balancer |check| 1000 -Rules per application load balancer |check| 100 -Target groups |check| 3000 -========================================== =============== ======= ==== +========================================== =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +========================================== =============== ======== ======= ==== +Application load balancers |check| |check| 20 +Certificates per application load balancer 25 +Classic load balancers |check| |check| 20 +Listeners per application load balancer |check| 50 +Listeners per load balancer |check| 100 +Listeners per network load balancer |check| 50 +Network load balancers |check| 20 +Registered instances per load balancer |check| 1000 +Rules per application load balancer |check| 100 +Target groups |check| 3000 +========================================== =============== ======== ======= ==== .. _limits.ElastiCache: ElastiCache ------------ -======================== =============== ======= === -Limit Trusted Advisor API Default -======================== =============== ======= === -Nodes 100 -Nodes per Cluster 20 -Parameter Groups 20 -Security Groups 50 -Subnet Groups 50 -Subnets per subnet group 20 -======================== =============== ======= === +======================== =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +======================== =============== ======== ======= === +Nodes 100 +Nodes per Cluster 20 +Parameter Groups 20 +Security Groups 50 +Subnet Groups 50 +Subnets per subnet group 20 +======================== =============== ======== ======= === .. _limits.ElasticBeanstalk: ElasticBeanstalk ----------------- -==================== =============== ======= ==== -Limit Trusted Advisor API Default -==================== =============== ======= ==== -Application versions 1000 -Applications 75 -Environments 200 -==================== =============== ======= ==== +==================== =============== ======== ======= ==== +Limit Trusted Advisor Quotas API Default +==================== =============== ======== ======= ==== +Application versions 1000 +Applications 75 +Environments 200 +==================== =============== ======== ======= ==== .. _limits.Firehose: Firehose --------- -=========================== =============== ======= == -Limit Trusted Advisor API Default -=========================== =============== ======= == -Delivery streams per region 50 -=========================== =============== ======= == +=========================== =============== ======== ======= == +Limit Trusted Advisor Quotas API Default +=========================== =============== ======== ======= == +Delivery streams per region |check| 50 +=========================== =============== ======== ======= == .. _limits.IAM: IAM ---- -====================== =============== ======= ===== -Limit Trusted Advisor API Default -====================== =============== ======= ===== -Groups |check| |check| 300 -Instance profiles |check| |check| 1000 -Policies |check| |check| 1500 -Policy Versions In Use |check| 10000 -Roles |check| |check| 1000 -Server certificates |check| |check| 20 -Users |check| |check| 5000 -====================== =============== ======= ===== +====================== =============== ======== ======= ===== +Limit Trusted Advisor Quotas API Default +====================== =============== ======== ======= ===== +Groups |check| |check| |check| 300 +Instance profiles |check| |check| |check| 1000 +Policies |check| |check| |check| 1500 +Policy Versions In Use |check| 10000 +Roles |check| |check| |check| 1000 +Server certificates |check| |check| |check| 20 +Users |check| |check| |check| 5000 +====================== =============== ======== ======= ===== .. _limits.Lambda: Lambda ------- -===================================== =============== ======= ===== -Limit Trusted Advisor API Default -===================================== =============== ======= ===== -Code Size Unzipped (MiB) per Function |check| 250 -Code Size Zipped (MiB) per Function |check| 50 -Concurrent Executions |check| 1000 -Function Count None -Total Code Size (MiB) |check| 76800 -Unreserved Concurrent Executions |check| 1000 -===================================== =============== ======= ===== +===================================== =============== ======== ======= ===== +Limit Trusted Advisor Quotas API Default +===================================== =============== ======== ======= ===== +Code Size Unzipped (MiB) per Function |check| 250 +Code Size Zipped (MiB) per Function |check| 50 +Concurrent Executions |check| 1000 +Function Count None +Total Code Size (MiB) |check| 76800 +Unreserved Concurrent Executions |check| 1000 +===================================== =============== ======== ======= ===== .. _limits.RDS: RDS ---- -============================ =============== ======= ====== -Limit Trusted Advisor API Default -============================ =============== ======= ====== -DB Cluster Parameter Groups |check| |check| 50 -DB Clusters |check| |check| 40 -DB instances |check| |check| 40 -DB parameter groups |check| |check| 50 -DB security groups |check| |check| 25 -DB snapshots per user |check| 100 -Event Subscriptions |check| |check| 20 -Max auths per security group |check| |check| 20 -Option Groups |check| 20 -Read replicas per master |check| |check| 5 -Reserved Instances |check| 40 -Storage quota (GB) |check| |check| 100000 -Subnet Groups |check| |check| 50 -Subnets per Subnet Group |check| |check| 20 -VPC Security Groups 5 -============================ =============== ======= ====== +============================ =============== ======== ======= ====== +Limit Trusted Advisor Quotas API Default +============================ =============== ======== ======= ====== +DB Cluster Parameter Groups |check| |check| |check| 50 +DB Clusters |check| |check| |check| 40 +DB instances |check| |check| |check| 40 +DB parameter groups |check| |check| |check| 50 +DB security groups |check| |check| |check| 25 +DB snapshots per user |check| |check| 100 +Event Subscriptions |check| |check| |check| 20 +Max auths per security group |check| |check| |check| 20 +Option Groups |check| |check| 20 +Read replicas per master |check| |check| |check| 5 +Reserved Instances |check| |check| 40 +Storage quota (GB) |check| |check| |check| 100000 +Subnet Groups |check| |check| |check| 50 +Subnets per Subnet Group |check| |check| |check| 20 +VPC Security Groups 5 +============================ =============== ======== ======= ====== .. _limits.Redshift: Redshift --------- -========================= =============== ======= == -Limit Trusted Advisor API Default -========================= =============== ======= == -Redshift manual snapshots 20 -Redshift subnet groups 20 -========================= =============== ======= == +========================= =============== ======== ======= == +Limit Trusted Advisor Quotas API Default +========================= =============== ======== ======= == +Redshift manual snapshots 20 +Redshift subnet groups 20 +========================= =============== ======== ======= == .. _limits.Route53: @@ -505,54 +656,54 @@ set per-hosted zone, and can be increased by AWS support per-hosted zone. As such, each zone may have a different limit value. -================================ =============== ======= ===== -Limit Trusted Advisor API Default -================================ =============== ======= ===== -Record sets per hosted zone |check| 10000 -VPC associations per hosted zone |check| 100 -================================ =============== ======= ===== +================================ =============== ======== ======= ===== +Limit Trusted Advisor Quotas API Default +================================ =============== ======== ======= ===== +Record sets per hosted zone |check| 10000 +VPC associations per hosted zone |check| 100 +================================ =============== ======== ======= ===== .. _limits.S3: S3 --- -======= =============== ======= === -Limit Trusted Advisor API Default -======= =============== ======= === -Buckets 100 -======= =============== ======= === +======= =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +======= =============== ======== ======= === +Buckets 100 +======= =============== ======== ======= === .. _limits.SES: SES ---- -=================== =============== ======= === -Limit Trusted Advisor API Default -=================== =============== ======= === -Daily sending quota |check| |check| 200 -=================== =============== ======= === +=================== =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +=================== =============== ======== ======= === +Daily sending quota |check| |check| 200 +=================== =============== ======== ======= === .. _limits.VPC: VPC ---- -============================= =============== ======= === -Limit Trusted Advisor API Default -============================= =============== ======= === -Entries per route table 50 -Internet gateways |check| 5 -NAT Gateways per AZ 5 -Network ACLs per VPC 200 -Network interfaces per Region |check| 350 -Route tables per VPC 200 -Rules per network ACL 20 -Subnets per VPC 200 -VPCs |check| 5 -Virtual private gateways 5 -============================= =============== ======= === +============================= =============== ======== ======= === +Limit Trusted Advisor Quotas API Default +============================= =============== ======== ======= === +Entries per route table 50 +Internet gateways |check| |check| 5 +NAT Gateways per AZ 5 +Network ACLs per VPC 200 +Network interfaces per Region |check| 350 +Route tables per VPC 200 +Rules per network ACL 20 +Subnets per VPC 200 +VPCs |check| |check| 5 +Virtual private gateways 5 +============================= =============== ======== ======= === diff --git a/docs/source/python_usage.rst b/docs/source/python_usage.rst index 4a0549fb..48e112fb 100644 --- a/docs/source/python_usage.rst +++ b/docs/source/python_usage.rst @@ -34,7 +34,7 @@ We also import :py:mod:`pprint` to make the output nicer. >>> import logging >>> logging.basicConfig() >>> logger = logging.getLogger() - >>> + >>> >>> from awslimitchecker.checker import AwsLimitChecker >>> c = AwsLimitChecker() @@ -49,7 +49,7 @@ parameter to the class constructor: >>> import logging >>> logging.basicConfig() >>> logger = logging.getLogger() - >>> + >>> >>> from awslimitchecker.checker import AwsLimitChecker >>> c = AwsLimitChecker(region='us-west-2') @@ -98,6 +98,8 @@ this can be specified by the ``external_id`` parameter. All are strings: >>> external_id='myid' >>> ) +.. _python_usage.limit_overrides: + Setting a Limit Override +++++++++++++++++++++++++ @@ -186,7 +188,7 @@ In this particular case, there is no resource ID associated with the usage, beca .. code-block:: pycon >>> result['EC2']['Magnetic volume storage (TiB)'].get_criticals()[0].resource_id - >>> + >>> The usage is of the EC2 Volume resource type (where one exists, we use the `CloudFormation Resource Type strings `_ to identify resource types). @@ -227,19 +229,38 @@ crossed the critical threshold: >>> for usage in result['EC2']['Security groups per VPC'].get_criticals(): ... print(str(usage)) - ... + ... vpc-c300b9a6=100 Disabling Trusted Advisor ++++++++++++++++++++++++++ -To disable querying Trusted Advisor for limit information, simply call :py:meth:`~.AwsLimitChecker.get_limits` +To disable querying Trusted Advisor for limit information, call :py:meth:`~.AwsLimitChecker.get_limits` or :py:meth:`~.AwsLimitChecker.check_thresholds` with ``use_ta=False``: .. code-block:: pycon >>> result = c.check_thresholds(use_ta=False) +.. _python_usage.disabling_service_quotas: + +Disabling Service Quotas +++++++++++++++++++++++++ + +To disable querying the Service Quotas service for current limits, pass ``skip_quotas=True`` +in to the :py:class:`~.AwsLimitChecker` class constructor: + +.. code-block:: python + + checker = AwsLimitChecker(skip_quotas=True) + +.. _python_usage.partitions: + +Partitions and Trusted Advisor Regions +++++++++++++++++++++++++++++++++++++++ + +awslimitchecker currently supports operating against non-standard `partitions `_, such as GovCloud and AWS China (Beijing). Partition names, as seen in the ``partition`` field of ARNs, can be specified with the ``role_partition`` keyword argument to the :py:class:`~.AwsLimitChecker` class. Similarly, the region name to use for the ``support`` API for Trusted Advisor can be specified with the ``ta_api_region`` keyword argument to the :py:class:`~.AwsLimitChecker` class. + Skipping Specific Services ++++++++++++++++++++++++++ @@ -254,6 +275,13 @@ To remove the Firehose and EC2 services: c.remove_services(['Firehose', 'EC2']) +.. _python_usage.throttling: + +Handling Throttling and Rate Limiting ++++++++++++++++++++++++++++++++++++++ + +See :ref:`CLI Usage - Handling Throttling and Rate Limiting `; this is handled the same way in Python, though you'd likely set the environment variables using ``os.environ`` instead of exporting them outside of Python. + Logging ------- @@ -310,11 +338,11 @@ multiple critical thresholds crossed. >>> import logging >>> logging.basicConfig() >>> logger = logging.getLogger() - >>> + >>> >>> from awslimitchecker.checker import AwsLimitChecker >>> c = AwsLimitChecker() >>> result = c.check_thresholds() - >>> + >>> >>> have_critical = False >>> for service, svc_limits in result.items(): ... for limit_name, limit in svc_limits.items(): @@ -337,7 +365,7 @@ multiple critical thresholds crossed. ... l=limit.get_limit(), ... ) ... ) - ... + ... CRITICAL:root:EC2 'Magnetic volume storage (TiB)' usage (23.417) exceeds critical threshold (limit=20) CRITICAL:root:EC2 'Running On-Demand EC2 instances' usage (97) exceeds critical threshold (limit=20) WARNING:root:EC2 'Security groups per VPC' usage (vpc-c300b9a6=96) exceeds warning threshold (limit=100) @@ -345,6 +373,6 @@ multiple critical thresholds crossed. CRITICAL:root:EC2 'EC2-Classic Elastic IPs' usage (5) exceeds critical threshold (limit=5) >>> if have_critical: ... raise SystemExit(1) - ... + ... (awslimitchecker)$ echo $? 1 diff --git a/setup.py b/setup.py index 93f04647..7536d8a2 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ long_description = file.read() requires = [ - 'boto3>=1.4.6', - 'botocore>=1.6.0', + 'boto3>=1.9.175', + 'botocore>=1.12.175', 'termcolor>=1.1.0', 'python-dateutil>=2.4.2', 'versionfinder>=0.1.1', @@ -54,7 +54,7 @@ ] classifiers = [ - 'Development Status :: 4 - Beta', + 'Development Status :: 6 - Mature', 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', @@ -63,13 +63,11 @@ 'Natural Language :: English', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Internet', 'Topic :: System :: Monitoring', ] diff --git a/tox.ini b/tox.ini index 88bd4090..41546edf 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37,pypy,pypy3,docs,localdocs,integration,integration3,docker +envlist = py27,py34,py35,py36,py37,py38,pypy,pypy3,docs,localdocs,integration,integration3,docker [testenv] deps = @@ -88,7 +88,7 @@ deps = sphinx sphinx_rtd_theme onetimepass==1.0.1 -basepython = python2.7 +basepython = python3.7 commands = python --version virtualenv --version @@ -116,7 +116,7 @@ deps = sphinx sphinx_rtd_theme onetimepass==1.0.1 -basepython = python2.7 +basepython = python3.7 commands = python --version virtualenv --version @@ -161,7 +161,7 @@ commands = deps = {[testenv:integration]deps} passenv = {[testenv:integration]passenv} setenv = {[testenv:integration]setenv} -basepython = python3.6 +basepython = python3.7 sitepackages = False whitelist_externals = env test commands = {[testenv:integration]commands}