Merge pull request #450 from /issues/449

fixes #449 - 8.0.0 release

.coveragerc

@@ -13,3 +13,4 @@ exclude_lines =
     raise NotImplementedError
     except ImportError:
     pragma: no cover
+    .*# nocoverage.*

.travis.yml

@@ -15,17 +15,21 @@ matrix:
       env: TOXENV=py37
       dist: xenial
       sudo: true
+    - python: "3.8"
+      env: TOXENV=py38
+      dist: xenial
+      sudo: true
     - python: "3.7"
       env: TOXENV=docker
       dist: xenial
       sudo: true
       services:
       - docker
-    - python: "2.7"
+    - python: "3.7"
       env: TOXENV=docs
     - python: "2.7"
       env: TOXENV=integration
-    - python: "3.6"
+    - python: "3.7"
       env: TOXENV=integration3
 install:
 - git config --global user.email "[email protected]"

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3-alpine
+FROM python:3.8-alpine
 
 ARG git_version
 

README.rst

@@ -17,10 +17,6 @@ awslimitchecker
    :alt: GitHub Open Issues
    :target: https://github.com/jantman/awslimitchecker/issues
 
-.. image:: https://badge.waffle.io/jantman/awslimitchecker.png?label=ready&title=Ready
-   :target: https://waffle.io/jantman/awslimitchecker
-   :alt: 'Stories in Ready - waffle.io'
-
 .. image:: http://www.repostatus.org/badges/1.1.0/active.svg
    :alt: Project Status: Active - The project has reached a stable, usable state and is being actively developed.
    :target: http://www.repostatus.org/#active
@@ -35,10 +31,6 @@ Master:
    :target: http://travis-ci.org/jantman/awslimitchecker
    :alt: travis-ci for master branch
 
-.. image:: https://landscape.io/github/jantman/awslimitchecker/master/landscape.svg?style=flat
-   :target: https://landscape.io/github/jantman/awslimitchecker/master
-   :alt: Code Health
-
 .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=master
    :target: https://codecov.io/github/jantman/awslimitchecker?branch=master
    :alt: coverage report for master branch
@@ -53,10 +45,6 @@ Develop:
    :target: http://travis-ci.org/jantman/awslimitchecker
    :alt: travis-ci for develop branch
 
-.. image:: https://landscape.io/github/jantman/awslimitchecker/develop/landscape.svg?style=flat
-   :target: https://landscape.io/github/jantman/awslimitchecker/develop
-   :alt: Code Health
-
 .. image:: https://codecov.io/github/jantman/awslimitchecker/coverage.svg?branch=develop
    :target: https://codecov.io/github/jantman/awslimitchecker?branch=develop
    :alt: coverage report for develop branch
@@ -70,9 +58,11 @@ A script and python module to check your AWS service limits and usage, and warn
 Users building out scalable services in Amazon AWS often run into AWS' `service limits <http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html>`_ -
 often at the least convenient time (i.e. mid-deploy or when autoscaling fails). Amazon's `Trusted Advisor <https://aws.amazon.com/premiumsupport/trustedadvisor/>`_
 can help this, but even the version that comes with Business and Enterprise support only monitors a small subset of AWS limits
-and only alerts *weekly*. awslimitchecker provides a command line script and reusable package that queries your current
-usage of AWS resources and compares it to limits (hard-coded AWS defaults that you can override, API-based limits where available, or data from Trusted
-Advisor where available), notifying you when you are approaching or at your limits.
+and only alerts *weekly*. The new Service Quotas service can also help with this, but relies on CloudWatch alarms per-limit to notify
+you when you approach your limits; this cannot easily scale to the hundreds of current service limits. awslimitchecker provides a command line
+script and reusable Python package that queries your current usage of AWS resources and compares it to limits (hard-coded AWS defaults that you
+can override, API-based limits where available, Service Quotas data where available, or data from Trusted Advisor where available), notifying
+you when you are approaching or at your limits.
 
 Full project documentation for the latest release is available at `http://awslimitchecker.readthedocs.io/en/latest/ <http://awslimitchecker.readthedocs.io/en/latest/>`_.
 
@@ -94,6 +84,7 @@ What It Does
 - Define custom thresholds per-limit
 - where possible, pull current limits from Trusted Advisor API
 - where possible, pull current limits from each service's API (for services that provide this information)
+- where possible, pull current limits from the Service Quotas service
 - Supports explicitly setting the AWS region
 - Supports using `STS <http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html>`_ to assume roles in other accounts, including using ``external_id``.
 - Optionally refresh Trusted Advisor "Service Limits" check before polling
@@ -109,7 +100,7 @@ Requirements
 
 **Either Docker in order to run via the** `docker image <http://awslimitchecker.readthedocs.io/en/latest/docker.html>`__, **or:**
 
-* Python 2.7 or 3.4+. Python 2.6 and 3.3 are no longer supported.
+* Python 3.5 or newer. Python 2.7 will not be supported as of January 1, 2010.
 * Python `VirtualEnv <http://www.virtualenv.org/>`_ and ``pip`` (recommended installation method; your OS/distribution should have packages for these)
 * `boto3 <http://boto3.readthedocs.org/>`_ >= 1.4.6 and its dependency `botocore <https://botocore.readthedocs.io/en/latest/>`_ >= 1.6.0.
 

awslimitchecker/checker.py

@@ -42,20 +42,30 @@
 from .trustedadvisor import TrustedAdvisor
 from .version import _get_version_info
 from .utils import _get_latest_version
+from .quotas import ServiceQuotasClient
 import boto3
 import sys
 import logging
+import warnings
 
 logger = logging.getLogger(__name__)
 
+warnings.filterwarnings(
+    action="always", category=DeprecationWarning, module=__name__
+)
+warnings.filterwarnings(
+    action="always", category=PendingDeprecationWarning, module=__name__
+)
+
 
 class AwsLimitChecker(object):
 
     def __init__(self, warning_threshold=80, critical_threshold=99,
                  profile_name=None, account_id=None, account_role=None,
-                 region=None, external_id=None, mfa_serial_number=None,
-                 mfa_token=None, ta_refresh_mode=None, ta_refresh_timeout=None,
-                 check_version=True):
+                 role_partition='aws', region=None, external_id=None,
+                 mfa_serial_number=None, mfa_token=None, ta_refresh_mode=None,
+                 ta_refresh_timeout=None, ta_api_region='us-east-1',
+                 check_version=True, skip_quotas=False):
         """
         Main AwsLimitChecker class - this should be the only externally-used
         portion of awslimitchecker.
@@ -89,6 +99,10 @@ def __init__(self, warning_threshold=80, critical_threshold=99,
         :param region: AWS region name to connect to
         :type region: str
         :type account_role: str
+        :param role_partition: `AWS role partition <https://docs.aws.amazon.com/
+          general/latest/gr/aws-arns-and-namespaces.html>`_
+          for the account_role to connect via STS
+        :type role_partition: str
         :param external_id: (optional) the `External ID <http://docs.aws.amazon.
           com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html>`_
           string to use when assuming a role via STS.
@@ -116,9 +130,16 @@ def __init__(self, warning_threshold=80, critical_threshold=99,
           parameter is not None, only wait up to this number of seconds for the
           refresh to finish before continuing on anyway.
         :type ta_refresh_timeout: :py:class:`int` or :py:data:`None`
+        :param ta_api_region: The AWS region used for calls to the
+          TrustedAdvisor API. This is always us-east-1 for
+          non GovCloud accounts.
+        :type ta_api_region: str
         :param check_version: Whether or not to check for latest version of
           awslimitchecker on PyPI during instantiation.
         :type check_version: bool
+        :param skip_quotas: If set to True, do not connect to Service Quotas
+          service or use it to obtain current limits.
+        :type skip_quotas: bool
         """
         # ###### IMPORTANT license notice ##########
         # Pursuant to Sections 5(b) and 13 of the GNU Affero General Public
@@ -151,11 +172,13 @@ def __init__(self, warning_threshold=80, critical_threshold=99,
                     ' is %s; please consider upgrading.', self.vinfo.release,
                     latest_ver
                 )
+        self._check_python_version()
         self.warning_threshold = warning_threshold
         self.critical_threshold = critical_threshold
         self.profile_name = profile_name
         self.account_id = account_id
         self.account_role = account_role
+        self.role_partition = role_partition
         self.external_id = external_id
         self.mfa_serial_number = mfa_serial_number
         self.mfa_token = mfa_token
@@ -164,15 +187,64 @@ def __init__(self, warning_threshold=80, critical_threshold=99,
         self.services = {}
 
         boto_conn_kwargs = self._boto_conn_kwargs
+        self._quotas_client = None
+        if not skip_quotas:
+            self._quotas_client = ServiceQuotasClient(boto_conn_kwargs)
         for sname, cls in _services.items():
             self.services[sname] = cls(warning_threshold,
                                        critical_threshold,
-                                       boto_conn_kwargs)
+                                       boto_conn_kwargs,
+                                       self._quotas_client)
 
         self.ta = TrustedAdvisor(self.services,
                                  boto_conn_kwargs,
                                  ta_refresh_mode=ta_refresh_mode,
-                                 ta_refresh_timeout=ta_refresh_timeout)
+                                 ta_refresh_timeout=ta_refresh_timeout,
+                                 ta_api_region=ta_api_region)
+
+    def _check_python_version(self):
+        """
+        Check that we are running under a supported Python version, and emit a
+        warning otherwise.
+        """
+        if sys.version_info[:2] == (2, 7):  # nocoverage
+            warnings.warn(
+                'awslimitchecker has detected that it is running under Python '
+                '2.7. This will no longer be supported as of January 1, 2020. '
+                'Please update to a newer Python version (>= 3.5) or switch '
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>',
+                PendingDeprecationWarning
+            )
+        elif sys.version_info[:2] == (3, 4):  # nocoverage
+            warnings.warn(
+                'awslimitchecker has detected that it is running under Python '
+                '3.4. This will no longer be supported as of January 1, 2020. '
+                'Please update to a newer Python version (>= 3.5) or switch '
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>',
+                PendingDeprecationWarning
+            )
+        elif (
+            sys.version_info[0] < 3 or
+            sys.version_info[0] == 3 and sys.version_info[1] < 4
+        ):  # nocoverage
+            warnings.warn(
+                'awslimitchecker has detected that it is running under Python '
+                '%d.%d. This version has reached end-of-life and is no longer '
+                'supported by awslimitchecker, and may not function correctly. '
+                'Please update to a newer Python version (>= 3.5) or switch '
+                'to running via the official Docker image. For further '
+                'information, please see the awslimitchecker 8.0.0 changelog '
+                'at <https://awslimitchecker.readthedocs.io/en/latest/changes.'
+                'html#changelog-8-0-0>'
+                '' % (sys.version_info[0], sys.version_info[1]),
+                DeprecationWarning
+            )
 
     @property
     def _boto_conn_kwargs(self):
@@ -277,6 +349,7 @@ def get_limits(self, service=None, use_ta=True):
         for sname, cls in to_get.items():
             if hasattr(cls, '_update_limits_from_api'):
                 cls._update_limits_from_api()
+            cls._update_service_quotas()
             res[sname] = cls.get_limits()
         return res
 
@@ -306,7 +379,11 @@ def _get_sts_token(self):
         """
         logger.debug("Connecting to STS in region %s", self.region)
         sts = boto3.client('sts', region_name=self.region)
-        arn = "arn:aws:iam::%s:role/%s" % (self.account_id, self.account_role)
+        arn = "arn:%s:iam::%s:role/%s" % (
+            self.role_partition,
+            self.account_id,
+            self.account_role
+        )
         logger.debug("STS assume role for %s", arn)
         assume_kwargs = {
             'RoleArn': arn,
@@ -351,6 +428,7 @@ def find_usage(self, service=None, use_ta=True):
         for cls in to_get.values():
             if hasattr(cls, '_update_limits_from_api'):
                 cls._update_limits_from_api()
+            cls._update_service_quotas()
             logger.debug("Finding usage for service: %s", cls.service_name)
             cls.find_usage()
 
@@ -551,6 +629,7 @@ def check_thresholds(self, service=None, use_ta=True):
         for sname, cls in to_get.items():
             if hasattr(cls, '_update_limits_from_api'):
                 cls._update_limits_from_api()
+            cls._update_service_quotas()
             tmp = cls.check_thresholds()
             if len(tmp) > 0:
                 res[sname] = tmp
@@ -569,6 +648,7 @@ def get_required_iam_policy(self):
         :rtype: dict
         """
         required_actions = [
+            'servicequotas:ListServiceQuotas',
             'support:*',
             'trustedadvisor:Describe*',
             'trustedadvisor:RefreshCheck'

awslimitchecker/connectable.py

@@ -37,8 +37,10 @@
 ################################################################################
 """
 
+import os
 import logging
 import boto3
+from botocore.config import Config
 
 logger = logging.getLogger(__name__)
 
@@ -66,12 +68,38 @@ def __init__(self, creds_dict):
 
 
 class Connectable(object):
-
     """
     Mix-in helper class for connecting to AWS APIs. Centralizes logic of
     connecting via regions and/or STS.
     """
 
+    @property
+    def _max_retries_config(self):
+        """
+        If a ``BOTO_MAX_RETRIES_<self.api_name>`` environment variable is set,
+        return a new ``botocore.config.Config`` instance using that number
+        as the retries max_attempts value.
+
+        :rtype: ``botocore.config.Config`` or None
+        """
+        key = 'BOTO_MAX_RETRIES_%s' % self.api_name
+        if key not in os.environ:
+            return None
+        try:
+            max_retries = int(os.environ[key])
+        except Exception:
+            logger.error(
+                'ERROR: Found "%s" environment variable, but unable to '
+                'parse value "%s" to an integer.', key, os.environ[key]
+            )
+            return None
+        logger.debug(
+            'Setting explicit botocore retry config with max_attempts=%d '
+            'for "%s" API based on %s environment variable.',
+            max_retries, self.api_name, key
+        )
+        return Config(retries={'max_attempts': max_retries})
+
     def connect(self):
         """
         Connect to an AWS API via boto3 low-level client and set ``self.conn``
@@ -84,7 +112,9 @@ def connect(self):
         """
         if self.conn is not None:
             return
-        kwargs = self._boto3_connection_kwargs
+        kwargs = dict(self._boto3_connection_kwargs)
+        if self._max_retries_config is not None:
+            kwargs['config'] = self._max_retries_config
         self.conn = boto3.client(self.api_name, **kwargs)
         logger.info("Connected to %s in region %s",
                     self.api_name, self.conn._client_config.region_name)
@@ -102,7 +132,9 @@ def connect_resource(self):
         """
         if self.resource_conn is not None:
             return
-        kwargs = self._boto3_connection_kwargs
+        kwargs = dict(self._boto3_connection_kwargs)
+        if self._max_retries_config is not None:
+            kwargs['config'] = self._max_retries_config
         self.resource_conn = boto3.resource(self.api_name, **kwargs)
         logger.info("Connected to %s (resource) in region %s", self.api_name,
                     self.resource_conn.meta.client._client_config.region_name)