From f87c15e459b32001210bb0bd5d14a2f3df6e4c59 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Sat, 27 Jun 2020 23:28:08 +0900 Subject: [PATCH 1/3] Add more tests --- django_elastipymemcache/memcached.py | 9 +- tests/test_backend.py | 95 ++++++++++++++++++-- tests/test_protocol.py | 124 ++++++++++++--------------- 3 files changed, 149 insertions(+), 79 deletions(-) diff --git a/django_elastipymemcache/memcached.py b/django_elastipymemcache/memcached.py index 90b084f..cc676b3 100644 --- a/django_elastipymemcache/memcached.py +++ b/django_elastipymemcache/memcached.py @@ -113,11 +113,6 @@ def _cache(self): return self._client - def close(self, **kwargs): - # libmemcached manages its own connections. Don't call disconnect_all() - # as it resets the failover state and creates unnecessary reconnects. - pass - @invalidate_cache_after_error def get(self, *args, **kwargs): return super(ElastiPyMemCache, self).get(*args, **kwargs) @@ -137,3 +132,7 @@ def set_many(self, *args, **kwargs): @invalidate_cache_after_error def delete(self, *args, **kwargs): return super(ElastiPyMemCache, self).delete(*args, **kwargs) + + @invalidate_cache_after_error + def delete_many(self, *args, **kwargs): + return super(ElastiPyMemCache, self).delete_many(*args, **kwargs) diff --git a/tests/test_backend.py b/tests/test_backend.py index 4627cab..72edf68 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,12 +1,33 @@ import socket -import sys +from unittest.mock import patch, Mock -from nose.tools import eq_ +try: + import cPickle as pickle +except ImportError: + import pickle -if sys.version < '3': - from mock import patch, Mock -else: - from unittest.mock import patch, Mock + +from django.core.cache import InvalidCacheBackendError +from nose.tools import ( + eq_, + raises, +) + + +@raises(InvalidCacheBackendError) +def test_multiple_servers(): + from django_elastipymemcache.memcached import ( + ElastiPyMemCache, + ) + ElastiPyMemCache('h1:0,h2:0', {}) + + +@raises(InvalidCacheBackendError) +def test_wrong_server_format(): + from django_elastipymemcache.memcached import ( + ElastiPyMemCache, + ) + ElastiPyMemCache('h', {}) @patch('django_elastipymemcache.memcached.get_cluster_info') @@ -64,6 +85,14 @@ def test_node_info_cache(get_cluster_info): 'h', '0', False, socket._GLOBAL_DEFAULT_TIMEOUT) +@patch('django_elastipymemcache.memcached.get_cluster_info') +def test_failed_to_connect_servers(get_cluster_info): + from django_elastipymemcache.memcached import ElastiPyMemCache + backend = ElastiPyMemCache('h:0', {}) + get_cluster_info.side_effect = OSError() + eq_(backend.get_cluster_nodes(), []) + + @patch('django_elastipymemcache.memcached.get_cluster_info') def test_invalidate_cache(get_cluster_info): from django_elastipymemcache.memcached import ElastiPyMemCache @@ -149,3 +178,57 @@ def test_client_get_many(get_cluster_info): 'key2': 1509111630.048594, }, ) + + +@patch('django_elastipymemcache.memcached.get_cluster_info') +def test_client_set_many(get_cluster_info): + from django_elastipymemcache.memcached import ElastiPyMemCache + + servers = [('h1', 0), ('h2', 0)] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPyMemCache('h:0', {}) + ret = backend.set_many({'key1': 'value1', 'key2': 'value2'}) + eq_(ret, ['key1', 'key2']) + + +@patch('django_elastipymemcache.memcached.get_cluster_info') +def test_client_delete(get_cluster_info): + from django_elastipymemcache.memcached import ElastiPyMemCache + + servers = [('h1', 0), ('h2', 0)] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPyMemCache('h:0', {}) + ret = backend.delete('key1') + eq_(ret, None) + + +@patch('django_elastipymemcache.memcached.get_cluster_info') +def test_client_delete_many(get_cluster_info): + from django_elastipymemcache.memcached import ElastiPyMemCache + + servers = [('h1', 0), ('h2', 0)] + get_cluster_info.return_value = { + 'nodes': servers + } + + backend = ElastiPyMemCache('h:0', {}) + ret = backend.delete_many(['key1', 'key2']) + eq_(ret, None) + + +def test_serialize_pickle(): + from django_elastipymemcache.memcached import serialize_pickle + eq_(serialize_pickle('key', 'str'), ('str', 1)) + eq_(serialize_pickle('key', 0), (pickle.dumps(0), 2)) + + +def test_deserialize_pickle(): + from django_elastipymemcache.memcached import deserialize_pickle + eq_(deserialize_pickle('key', 'str', 1), 'str') + eq_(deserialize_pickle('key', pickle.dumps(0), 2), 0) diff --git a/tests/test_protocol.py b/tests/test_protocol.py index a536238..bc6a3af 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -4,7 +4,10 @@ WrongProtocolData, get_cluster_info, ) -from nose.tools import eq_, raises +from nose.tools import ( + eq_, + raises, +) if sys.version < '3': from mock import patch, call, MagicMock @@ -12,47 +15,32 @@ from unittest.mock import patch, call, MagicMock -TEST_PROTOCOL_1_READ_UNTIL = [ - b'VERSION 1.4.14', -] - -TEST_PROTOCOL_1_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_2_READ_UNTIL = [ - b'VERSION 1.4.13', -] - -TEST_PROTOCOL_2_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_3_READ_UNTIL = [ - b'VERSION 1.4.14 (Ubuntu)', -] - -TEST_PROTOCOL_3_EXPECT = [ - (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA -] - -TEST_PROTOCOL_4_READ_UNTIL = [ - b'VERSION 1.4.34', -] - -TEST_PROTOCOL_4_EXPECT = [ - (0, None, b'ERROR\r\n'), -] +# https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/AutoDiscovery.AddingToYourClientLibrary.html +EXAMPLE_RESPONSE = ( + b'CONFIG cluster 0 147\r\n' + b'12\n' + b'myCluster.pc4ldq.0001.use1.cache.amazonaws.com|10.82.235.120|11211 ' + b'myCluster.pc4ldq.0002.use1.cache.amazonaws.com|10.80.249.27|11211\n\r\n' + b'END\r\n' +) @patch('django_elastipymemcache.cluster_utils.Telnet') def test_happy_path(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_1_EXPECT + client.read_until.side_effect = [ + b'VERSION 1.4.14', + ] + client.expect.side_effect = [ + (0, None, EXAMPLE_RESPONSE), # NOQA + ] info = get_cluster_info('', 0) - eq_(info['version'], 1) - eq_(info['nodes'], [('ip', 11211), ('host', 11211)]) + eq_(info['version'], 12) + eq_(info['nodes'], [('10.82.235.120', 11211), ('10.80.249.27', 11211)]) + client.write.assert_has_calls([ + call(b'version\n'), + call(b'config get cluster\n'), + ]) @raises(WrongProtocolData) @@ -62,10 +50,14 @@ def test_bad_protocol(): @patch('django_elastipymemcache.cluster_utils.Telnet') -def test_last_versions(Telnet): +def test_ubuntu_protocol(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_1_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_1_EXPECT + client.read_until.side_effect = [ + b'VERSION 1.4.14 (Ubuntu)', + ] + client.expect.side_effect = [ + (0, None, b'CONFIG cluster 0 138\r\n1\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA + ] get_cluster_info('', 0) client.write.assert_has_calls([ call(b'version\n'), @@ -73,44 +65,40 @@ def test_last_versions(Telnet): ]) +@raises(WrongProtocolData) @patch('django_elastipymemcache.cluster_utils.Telnet') -def test_prev_versions(Telnet): +def test_no_configuration_protocol_support_with_errors(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_2_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_2_EXPECT - get_cluster_info('', 0) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'get AmazonElastiCache:cluster\n'), - ]) + client.read_until.side_effect = [ + b'VERSION 1.4.34', + ] + client.expect.side_effect = [ + (0, None, b'ERROR\r\n'), + ] + get_cluster_info('test', 0) +@raises(WrongProtocolData) @patch('django_elastipymemcache.cluster_utils.Telnet') -def test_ubuntu_protocol(Telnet): +def test_hoge(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_3_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_3_EXPECT - - # try: - # get_cluster_info('', 0) - # except WrongProtocolData: - # raise AssertionError('Raised WrongProtocolData with Ubuntu version.') - get_cluster_info('', 0) - - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) + client.read_until.side_effect = [ + b'VERSION 1.4.34', + ] + client.expect.side_effect = [ + (0, None, b'CONFIG cluster 0 138\r\nfail\nhost|ip|11211 host||11211\n\r\nEND\r\n'), # NOQA + ] + get_cluster_info('test', 0) @raises(WrongProtocolData) @patch('django_elastipymemcache.cluster_utils.Telnet') -def test_no_configuration_protocol_support_with_errors(Telnet): +def test_no_configuration_protocol_support_with_errors_but_ignored(Telnet): client = Telnet.return_value - client.read_until.side_effect = TEST_PROTOCOL_4_READ_UNTIL - client.expect.side_effect = TEST_PROTOCOL_4_EXPECT + client.read_until.side_effect = [ + b'VERSION 1.4.34', + ] + client.expect.side_effect = [ + (0, None, b'CONFIG cluster 0 138\r\n1\nfail\n\r\nEND\r\n'), # NOQA + ] get_cluster_info('test', 0) - client.write.assert_has_calls([ - call(b'version\n'), - call(b'config get cluster\n'), - ]) From 915e31b41dc7a6267f2cf5f736ece576b879bf23 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Sat, 27 Jun 2020 23:29:30 +0900 Subject: [PATCH 2/3] Bump version to 1.3.1 --- django_elastipymemcache/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elastipymemcache/__init__.py b/django_elastipymemcache/__init__.py index 619230e..528a100 100644 --- a/django_elastipymemcache/__init__.py +++ b/django_elastipymemcache/__init__.py @@ -1,2 +1,2 @@ -VERSION = (1, 3, 0) +VERSION = (1, 3, 1) __version__ = '.'.join(map(str, VERSION)) From 1fefcd0917ecde8f34fdb1c6ce80d980378aebfb Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Sun, 28 Jun 2020 00:06:13 +0900 Subject: [PATCH 3/3] Remove super's args --- django_elastipymemcache/client.py | 2 +- django_elastipymemcache/cluster_utils.py | 5 +++-- django_elastipymemcache/memcached.py | 18 +++++++++--------- tests/test_backend.py | 3 +-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/django_elastipymemcache/client.py b/django_elastipymemcache/client.py index 0f67743..c7392a5 100644 --- a/django_elastipymemcache/client.py +++ b/django_elastipymemcache/client.py @@ -4,7 +4,7 @@ class Client(HashClient): def get_many(self, keys, gets=False, *args, **kwargs): # pymemcache's HashClient may returns {'key': False} - end = super(Client, self).get_many(keys, gets, *args, **kwargs) + end = super().get_many(keys, gets, *args, **kwargs) return {key: end.get(key) for key in end if end.get(key)} diff --git a/django_elastipymemcache/cluster_utils.py b/django_elastipymemcache/cluster_utils.py index 15f2d9f..aeb0473 100644 --- a/django_elastipymemcache/cluster_utils.py +++ b/django_elastipymemcache/cluster_utils.py @@ -15,8 +15,9 @@ class WrongProtocolData(ValueError): in telnet protocol """ def __init__(self, cmd, response): - super(WrongProtocolData, self).__init__( - 'Unexpected response {} for command {}'.format(response, cmd)) + super().__init__( + 'Unexpected response {} for command {}'.format(response, cmd), + ) def get_cluster_info( diff --git a/django_elastipymemcache/memcached.py b/django_elastipymemcache/memcached.py index cc676b3..66b8ff1 100644 --- a/django_elastipymemcache/memcached.py +++ b/django_elastipymemcache/memcached.py @@ -13,7 +13,7 @@ from django.core.cache import InvalidCacheBackendError from django.core.cache.backends.memcached import BaseMemcachedCache -from . import client as pyMemcache_client +from . import client as pymemcache_client from .cluster_utils import get_cluster_info @@ -53,10 +53,10 @@ class ElastiPyMemCache(BaseMemcachedCache): it used pyMemcache """ def __init__(self, server, params): - super(ElastiPyMemCache, self).__init__( + super().__init__( server, params, - library=pyMemcache_client, + library=pymemcache_client, value_not_found_exception=ValueError) if len(self._servers) > 1: raise InvalidCacheBackendError( @@ -115,24 +115,24 @@ def _cache(self): @invalidate_cache_after_error def get(self, *args, **kwargs): - return super(ElastiPyMemCache, self).get(*args, **kwargs) + return super().get(*args, **kwargs) @invalidate_cache_after_error def get_many(self, *args, **kwargs): - return super(ElastiPyMemCache, self).get_many(*args, **kwargs) + return super().get_many(*args, **kwargs) @invalidate_cache_after_error def set(self, *args, **kwargs): - return super(ElastiPyMemCache, self).set(*args, **kwargs) + return super().set(*args, **kwargs) @invalidate_cache_after_error def set_many(self, *args, **kwargs): - return super(ElastiPyMemCache, self).set_many(*args, **kwargs) + return super().set_many(*args, **kwargs) @invalidate_cache_after_error def delete(self, *args, **kwargs): - return super(ElastiPyMemCache, self).delete(*args, **kwargs) + return super().delete(*args, **kwargs) @invalidate_cache_after_error def delete_many(self, *args, **kwargs): - return super(ElastiPyMemCache, self).delete_many(*args, **kwargs) + return super().delete_many(*args, **kwargs) diff --git a/tests/test_backend.py b/tests/test_backend.py index 72edf68..2311a35 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -140,8 +140,7 @@ def test_client_get_many(get_cluster_info): ret = backend.get_many(['key2']) eq_(ret, {}) - with patch('django_elastipymemcache.client.Client.get_many'), \ - patch('pymemcache.client.hash.HashClient._safely_run_func') as p2: + with patch('pymemcache.client.hash.HashClient._safely_run_func') as p2: p2.return_value = { ':1:key3': 1509111630.048594 }