Merge pull request #278 from /issues/277

Issues/277 - 0.9.0 release

CHANGES.rst

@@ -1,6 +1,31 @@
 Changelog
 =========
 
+0.9.0 (2017-06-11)
+------------------
+
+* `Issue #269 <https://github.com/jantman/awslimitchecker/issues/269>`_ - set Trusted
+  Advisor limit name overrides for some RDS limits that were recently added to TA, but
+  with different names than what awslimitchecker uses.
+* Fix bug `Issue #270 <https://github.com/jantman/awslimitchecker/issues/270>`_ -
+  do not count propagated routes towards the VPC "Entries per route table" limit,
+  per clarification in `VPC service limits documentation <http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html#limits_vpc>`_ ("This is the limit
+  for the number of non-propagated entries per route table.")
+* `PR #276 <https://github.com/jantman/awslimitchecker/pull/276>`_ /
+  `Issue #275 <https://github.com/jantman/awslimitchecker/issues/275>`_ - Add new
+  ``--skip-service`` CLI option and :py:meth:`~.AwsLimitChecker.remove_services` to allow
+  skipping of one or more specific services during runs. (Thanks to `tamsky <https://github.com/tamsky>`_)
+  for this contribution.)
+* `PR #274 <https://github.com/jantman/awslimitchecker/pull/274>`_ /
+  `Issue #273 <https://github.com/jantman/awslimitchecker/issues/273>`_ - Add support
+  for new ``i3`` EC2 Instance types.  (Thanks to `tamsky <https://github.com/tamsky>`_)
+  for this contribution.)
+* Fix broken docs build due to changes Intersphinx reference to ValueError in python2 docs
+* Add hack to ``docs/source/conf.py`` as workaround for https://github.com/sphinx-doc/sphinx/issues/3860
+* `Issue #267 <https://github.com/jantman/awslimitchecker/issues/267>`_ - Firehose is only
+  available in ``us-east-1``, ``us-west-2`` and ``eu-west-1``. Omit the traceback from the
+  log message for Firehose ``EndpointConnectionError`` and log at warning instead of error.
+
 0.8.0 (2017-03-11)
 ------------------
 

README.rst

@@ -21,7 +21,7 @@ awslimitchecker
    :target: https://waffle.io/jantman/awslimitchecker
    :alt: 'Stories in Ready - waffle.io'
 
-.. image:: http://www.repostatus.org/badges/0.1.0/active.svg
+.. image:: http://www.repostatus.org/badges/1.1.0/active.svg
    :alt: Project Status: Active - The project has reached a stable, usable state and is being actively developed.
    :target: http://www.repostatus.org/#active
 

awslimitchecker/checker.py

@@ -218,6 +218,27 @@ def get_project_url(self):
         """
         return self.vinfo.url
 
+    def remove_services(self, services_to_remove=[]):
+        """
+        Remove all service names specified in ``services_to_remove`` from
+        ``self.services``. This allows explicitly removing certain services from
+        ever being checked or otherwise handled.
+
+        By default, the various methods that work on Services (i.e.
+        :py:meth:`~.get_limits`, :py:meth:`~.find_usage` and
+        :py:meth:`~.check_thresholds`) operate on either all known services,
+        or one specified service name at a time. This method allows you to
+        remove one or more problematic or undesirable services from the dict
+        of all services, and then operate on the remaining ones.
+
+        :param services_to_remove: the name(s) of one or more services to
+          permanently exclude from future calls to this instance
+        :type service_to_skip: list
+        """
+        for sname in services_to_remove:
+            logger.warning('Skipping service: %s', sname)
+            self.services.pop(sname, None)
+
     def get_limits(self, service=None, use_ta=True):
         """
         Return all :py:class:`~.AwsLimit` objects for the given
@@ -351,8 +372,8 @@ def set_limit_overrides(self, override_dict, override_ta=True):
         :param override_ta: whether or not to use this value even if Trusted
           Advisor supplies limit information
         :type override_ta: bool
-        :raises: :py:exc:`exceptions.ValueError` if limit_name is not known to
-          the service instance
+        :raises: :py:exc:`ValueError` if limit_name is not known to the
+          service instance
         """
         for svc_name in override_dict:
             for lim_name in override_dict[svc_name]:
@@ -385,7 +406,8 @@ def set_limit_override(self, service_name, limit_name,
         :param override_ta: whether or not to use this value even if Trusted
           Advisor supplies limit information
         :type override_ta: bool
-        :raises: ValueError if limit_name is not known to the service instance
+        :raises: :py:exc:`ValueError` if limit_name is not known to the
+          service instance
         """
         self.services[service_name].set_limit_override(
             limit_name,

awslimitchecker/runner.py

@@ -101,6 +101,10 @@ def parse_args(self, argv):
         p.add_argument('-S', '--service', action='store', nargs='*',
                        help='perform action for only the specified service name'
                             '; see -s|--list-services for valid names')
+        p.add_argument('--skip-service', action='append', default=[],
+                       dest='skip_service',
+                       help='avoid performing actions for the specified service'
+                            ' name; see -s|--list-services for valid names')
         p.add_argument('-s', '--list-services', action='store_true',
                        default=False,
                        help='print a list of all AWS service types that '
@@ -365,6 +369,9 @@ def console_entry_point(self):
             ))
             raise SystemExit(0)
 
+        if len(args.skip_service) > 0:
+            self.checker.remove_services(args.skip_service)
+
         if len(args.limit) > 0:
             self.set_limit_overrides(args.limit)
 

awslimitchecker/services/ec2.py

@@ -318,28 +318,34 @@ def _get_limits_instances(self):
         # (On-Demand, Reserved, Spot)
         default_limits = (20, 20, 5)
         special_limits = {
-            'm4.4xlarge': (10, 20, 5),
-            'm4.10xlarge': (5, 20, 5),
-            'm4.16xlarge': (5, 20, 5),
             'c4.4xlarge': (10, 20, 5),
             'c4.8xlarge': (5, 20, 5),
             'cg1.4xlarge': (2, 20, 5),
-            'hi1.4xlarge': (2, 20, 5),
-            'hs1.8xlarge': (2, 20, 0),
             'cr1.8xlarge': (2, 20, 5),
-            'p2.xlarge': (1, 20, 5),
-            'p2.8xlarge': (1, 20, 5),
-            'p2.16xlarge': (1, 20, 5),
+            'd2.4xlarge': (10, 20, 5),
+            'd2.8xlarge': (5, 20, 5),
             'g2.2xlarge': (5, 20, 5),
             'g2.8xlarge': (2, 20, 5),
-            'r3.4xlarge': (10, 20, 5),
-            'r3.8xlarge': (5, 20, 5),
+            'hi1.4xlarge': (2, 20, 5),
+            'hs1.8xlarge': (2, 20, 0),
             'i2.xlarge': (8, 20, 0),
             'i2.2xlarge': (8, 20, 0),
             'i2.4xlarge': (4, 20, 0),
             'i2.8xlarge': (2, 20, 0),
-            'd2.4xlarge': (10, 20, 5),
-            'd2.8xlarge': (5, 20, 5),
+            'i3.large': (2, 20, 0),
+            'i3.xlarge': (2, 20, 0),
+            'i3.2xlarge': (2, 20, 0),
+            'i3.4xlarge': (2, 20, 0),
+            'i3.8xlarge': (2, 20, 0),
+            'i3.16xlarge': (2, 20, 0),
+            'm4.4xlarge': (10, 20, 5),
+            'm4.10xlarge': (5, 20, 5),
+            'm4.16xlarge': (5, 20, 5),
+            'p2.xlarge': (1, 20, 5),
+            'p2.8xlarge': (1, 20, 5),
+            'p2.16xlarge': (1, 20, 5),
+            'r3.4xlarge': (10, 20, 5),
+            'r3.8xlarge': (5, 20, 5),
         }
         limits = {}
         for i_type in self._instance_types():
@@ -649,6 +655,12 @@ def _instance_types(self):
             'i2.2xlarge',
             'i2.4xlarge',
             'i2.8xlarge',
+            'i3.large',
+            'i3.xlarge',
+            'i3.2xlarge',
+            'i3.4xlarge',
+            'i3.8xlarge',
+            'i3.16xlarge',
         ]
 
         PREV_STORAGE_TYPES = [

awslimitchecker/services/firehose.py

@@ -64,11 +64,11 @@ def find_usage(self):
             lim._reset_usage()
         try:
             self._find_delivery_streams()
-        except EndpointConnectionError:
-            logger.error(
-                'Caught exception when trying to use Firehose; '
-                'perhaps the Firehose service is not available in this region?',
-                exc_info=1,
+        except EndpointConnectionError as ex:
+            logger.warning(
+                'Caught exception when trying to use Firehose ('
+                'perhaps the Firehose service is not available in this '
+                'region?): %s', ex
             )
 
         self._have_usage = True

awslimitchecker/services/rds.py

@@ -147,7 +147,7 @@ def get_limits(self):
             40,
             self.warning_threshold,
             self.critical_threshold,
-            limit_type='AWS::RDS::DBInstance',
+            limit_type='AWS::RDS::DBInstance'
         )
         limits['Reserved Instances'] = AwsLimit(
             'Reserved Instances',
@@ -204,6 +204,7 @@ def get_limits(self):
             self.warning_threshold,
             self.critical_threshold,
             limit_type='AWS::RDS::DBSubnetGroup',
+            ta_limit_name='Subnet groups'
         )
         limits['Subnets per Subnet Group'] = AwsLimit(
             'Subnets per Subnet Group',
@@ -212,6 +213,7 @@ def get_limits(self):
             self.warning_threshold,
             self.critical_threshold,
             limit_type='AWS::RDS::DBSubnetGroup',
+            ta_limit_name='Subnets per subnet group'
         )
         limits['Option Groups'] = AwsLimit(
             'Option Groups',
@@ -228,6 +230,7 @@ def get_limits(self):
             self.warning_threshold,
             self.critical_threshold,
             limit_type='AWS::RDS::DBEventSubscription',
+            ta_limit_name='Event subscriptions'
         )
         limits['Read replicas per master'] = AwsLimit(
             'Read replicas per master',
@@ -254,6 +257,7 @@ def get_limits(self):
             self.warning_threshold,
             self.critical_threshold,
             limit_type='AWS::RDS::DBCluster',
+            ta_limit_name='Clusters'
         )
         limits['DB Cluster Parameter Groups'] = AwsLimit(
             'DB Cluster Parameter Groups',
@@ -262,6 +266,7 @@ def get_limits(self):
             self.warning_threshold,
             self.critical_threshold,
             limit_type='AWS::RDS::DBClusterParameterGroup',
+            ta_limit_name='Cluster parameter groups'
         )
         self.limits = limits
         return limits

awslimitchecker/services/vpc.py

@@ -124,8 +124,12 @@ def _find_usage_route_tables(self):
         for table in self.conn.describe_route_tables()['RouteTables']:
             tables[table['VpcId']] += 1
             # Entries per route table
+            routes = [
+                r for r in table['Routes']
+                if r['Origin'] != 'EnableVgwRoutePropagation'
+            ]
             self.limits['Entries per route table']._add_current_usage(
-                len(table['Routes']),
+                len(routes),
                 aws_type='AWS::EC2::RouteTable',
                 resource_id=table['RouteTableId']
             )

awslimitchecker/tests/services/result_fixtures.py

@@ -369,8 +369,16 @@ class VPC(object):
                         'State': 'active',
                         'Origin': 'CreateRouteTable'
                     },
-                    {'foo': 'bar', 'baz': 'blam'},
-                    {'foo': 'bar', 'baz': 'blam'},
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'CreateRoute'
+                    },
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'EnableVgwRoutePropagation'
+                    },
                 ],
                 'Associations': [
                     {
@@ -396,18 +404,42 @@ class VPC(object):
                 'RouteTableId': 'rt-2',
                 'VpcId': 'vpc-1',
                 'Routes': [
-                    {'foo': 'bar', 'baz': 'blam'},
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'CreateRouteTable'
+                    },
                 ],
             },
             {
                 'RouteTableId': 'rt-3',
                 'VpcId': 'vpc-2',
                 'Routes': [
-                    {'foo': 'bar', 'baz': 'blam'},
-                    {'foo': 'bar', 'baz': 'blam'},
-                    {'foo': 'bar', 'baz': 'blam'},
-                    {'foo': 'bar', 'baz': 'blam'},
-                    {'foo': 'bar', 'baz': 'blam'},
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'EnableVgwRoutePropagation'
+                    },
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'CreateRoute'
+                    },
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'CreateRoute'
+                    },
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'EnableVgwRoutePropagation'
+                    },
+                    {
+                        'foo': 'bar',
+                        'baz': 'blam',
+                        'Origin': 'CreateRouteTable'
+                    },
                 ],
             }
         ]

awslimitchecker/tests/services/test_ec2.py

@@ -76,11 +76,12 @@ def test_init(self):
     def test_instance_types(self):
         cls = _Ec2Service(21, 43)
         types = cls._instance_types()
-        assert len(types) == 70
+        assert len(types) == 76
         assert 't2.micro' in types
         assert 'r3.8xlarge' in types
         assert 'c3.large' in types
         assert 'i2.4xlarge' in types
+        assert 'i3.16xlarge' in types
         assert 'd2.2xlarge' in types
         assert 'g2.8xlarge' in types
         assert 'hs1.8xlarge' in types
@@ -134,7 +135,7 @@ def test_get_limits_all(self):
     def test_get_limits_instances(self):
         cls = _Ec2Service(21, 43)
         limits = cls._get_limits_instances()
-        assert len(limits) == 71
+        assert len(limits) == 77
         # check a random subset of limits
         t2_micro = limits['Running On-Demand t2.micro instances']
         assert t2_micro.default_limit == 20
@@ -148,6 +149,10 @@ def test_get_limits_instances(self):
         assert i2_8xlarge.default_limit == 2
         assert i2_8xlarge.limit_type == 'On-Demand instances'
         assert i2_8xlarge.limit_subtype == 'i2.8xlarge'
+        i3_16xlarge = limits['Running On-Demand i3.16xlarge instances']
+        assert i3_16xlarge.default_limit == 2
+        assert i3_16xlarge.limit_type == 'On-Demand instances'
+        assert i3_16xlarge.limit_subtype == 'i3.16xlarge'
         m4_16xlarge = limits['Running On-Demand m4.16xlarge instances']
         assert m4_16xlarge.default_limit == 5
         assert m4_16xlarge.limit_type == 'On-Demand instances'

awslimitchecker/tests/services/test_firehose.py

@@ -126,6 +126,7 @@ def test_find_usage_with_endpoint_connection_error(self):
         with patch('%s.logger' % self.pbm, autospec=True) as mock_logger:
             cls.find_usage()
         error_msg = (
-            'Caught exception when trying to use Firehose; '
-            'perhaps the Firehose service is not available in this region?')
-        assert call.error(error_msg, exc_info=1) in mock_logger.mock_calls
+            'Caught exception when trying to use Firehose ('
+            'perhaps the Firehose service is not available in this region?): '
+            '%s')
+        assert call.warning(error_msg, client_error) in mock_logger.mock_calls

awslimitchecker/tests/services/test_vpc.py

@@ -223,9 +223,9 @@ def test_find_usage_route_tables(self):
         assert entries[0].resource_id == 'rt-2'
         assert entries[0].get_value() == 1
         assert entries[1].resource_id == 'rt-1'
-        assert entries[1].get_value() == 3
+        assert entries[1].get_value() == 2
         assert entries[2].resource_id == 'rt-3'
-        assert entries[2].get_value() == 5
+        assert entries[2].get_value() == 3
         assert mock_conn.mock_calls == [
             call.describe_route_tables()
         ]