--- ./backends/rdbms_common.py 2011-07-15 09:03:38.000000000 +0100 +++ ./backends/rdbms_common.py 2011-08-18 14:08:00.741009498 +0100 @@ -58,7 +61,7 @@ from roundup import hyperdb, date, password, roundupdb, security, support from roundup.hyperdb import String, Password, Date, Interval, Link, \ Multilink, DatabaseError, Boolean, Number, Node -from roundup.backends import locking +from roundup.backends import locking, cache from roundup.support import reversed from roundup.i18n import _ @@ -179,12 +185,13 @@ # additional transaction support for external files and the like self.transactions = [] - # keep a cache of the N most recently retrieved rows of any kind - # (classname, nodeid) = row - self.cache_size = config.RDBMS_CACHE_SIZE - self.clearCache() - self.stats = {'cache_hits': 0, 'cache_misses': 0, 'get_items': 0, - 'filtering': 0} + self.cache = cache.get_cache(config) + self.stats = { + 'cache_hits': 0, + 'cache_misses': 0, + 'get_items': 0, + 'filtering': 0 + } # database lock self.lockfile = None @@ -193,8 +200,7 @@ self.open_connection() def clearCache(self): - self.cache = {} - self.cache_lru = [] + self.cache.clear_all() def getSessionManager(self): return Sessions(self) @@ -915,9 +926,7 @@ values[col] = None # clear this node out of the cache if it's in there - key = (classname, nodeid) - if key in self.cache: - self._cache_del(key) + self.cache.clear(self._key_for_cache((classname, nodeid))) # figure the values to insert vals = [] @@ -964,9 +973,7 @@ % (classname, nodeid, values)) # clear this node out of the cache if it's in there - key = (classname, nodeid) - if key in self.cache: - self._cache_del(key) + self.cache.clear(self._key_for_cache((classname, nodeid))) cl = self.classes[classname] props = cl.getprops() @@ -1112,16 +1119,15 @@ But for internal database operations we need them. """ # see if we have this node cached - key = (classname, nodeid) - if key in self.cache: - # push us back to the top of the LRU - self._cache_refresh(key) + node = self.cache.get(self._key_for_cache((classname, nodeid))) + if node is not None: if __debug__: self.stats['cache_hits'] += 1 + # return the cached information if fetch_multilinks: - self._materialize_multilinks(classname, nodeid, self.cache[key]) - return self.cache[key] + self._materialize_multilinks(classname, nodeid, node) + return node if __debug__: self.stats['cache_misses'] += 1 @@ -1158,8 +1166,7 @@ self._materialize_multilinks(classname, nodeid, node, mls) # save off in the cache - key = (classname, nodeid) - self._cache_save(key, node) + self.cache.set(self._key_for_cache((classname, nodeid)), node) if __debug__: self.stats['get_items'] += (time.time() - start_t) @@ -1178,8 +1185,7 @@ raise IndexError('%s has no node %s'%(classname, nodeid)) # see if we have this node cached - if (classname, nodeid) in self.cache: - del self.cache[(classname, nodeid)] + self.cache.clear(self._key_for_cache((classname, nodeid))) # see if there's any obvious commit actions that we should get rid of for entry in self.transactions[:]: @@ -1210,7 +1216,7 @@ """ # If this node is in the cache, then we do not need to go to # the database. (We don't consider this an LRU hit, though.) - if (classname, nodeid) in self.cache: + if self.cache.has_key(self._key_for_cache((classname, nodeid))): # Return 1, not True, to match the type of the result of # the SQL operation below. return 1 @@ -1434,10 +1444,10 @@ self.transactions = [] # clear the cache - self.clearCache() + self.cache.clear_all() def sql_close(self): logging.getLogger('roundup.hyperdb').info('close') @@ -1446,6 +1456,15 @@ self.indexer.close() self.sql_close() + logging.getLogger('hyperdb').debug('closing cache connection') + try: + self.cache.close() + except Exception, e: + logging.getLogger('hyperdb').exception("Error closing cache connection") + + + def _key_for_cache(self, (classname, nodeid)): + return "%s%s" % (classname, nodeid) # # The base Class class # --- ./configuration.py 2011-06-06 19:29:20.000000000 +0100 +++ ./configuration.py 2011-08-18 14:08:00.729009498 +0100 @@ -467,6 +468,7 @@ # compatibility - new options should *not* have aliases! SETTINGS = ( ("main", ( + (Option, "memcache", "", "Memcache uri for caching"), (FilePathOption, "database", "db", "Database directory path."), (FilePathOption, "templates", "html", "Path to the HTML templates directory."), --- /home/rxa/software/roundup-1.4.19/roundup//date.py 2011-06-06 19:29:20.000000000 +0100 +++ ./date.py 2011-08-19 10:57:20.225015443 +0100 @@ -280,6 +283,16 @@ except: raise ValueError, 'Unknown spec %r' % (spec,) + def __getstate__(self): + del self.__dict__["translator"] + del self.__dict__["_"] + del self.__dict__["ngettext"] + return self.__dict__ + + def __setstate__(self, d): + self.__dict__.update(d) + self.setTranslator(i18n) + def set(self, spec, offset=0, date_re=date_re, serialised_re=serialised_date_re, add_granularity=False): ''' set the date to the value in spec @@ -537,6 +550,8 @@ return Date((y, m, d, H, M, S, 0, 0, 0), translator=self.translator) def __deepcopy__(self, memo): + if not hasattr(self, "translator"): + self.setTranslator(i18n) return Date((self.year, self.month, self.day, self.hour, self.minute, self.second, 0, 0, 0), translator=self.translator) @@ -655,7 +670,19 @@ self.second = spec self.second = int(self.second) + def __getstate__(self): + del self.__dict__["translator"] + del self.__dict__["_"] + del self.__dict__["ngettext"] + return self.__dict__ + + def __setstate__(self, d): + self.__dict__.update(d) + self.setTranslator(i18n) + def __deepcopy__(self, memo): + if not hasattr(self, "translator"): + self.setTranslator(i18n) return Interval((self.sign, self.year, self.month, self.day, self.hour, self.minute, self.second), translator=self.translator) --- backends/cache.py 1970-01-01 01:00:00.000000000 +0100 +++ backends/cache.py 2011-08-18 14:08:00.737009498 +0100 @@ -0,0 +1,253 @@ +"""A module defining key-value based cache objects. + +There are two types of cache: + :class: `DefaultNodeCache` + A simple in-memory cache using an LRU. + + :class: `MemcacheNodeCache` + A cache using on ``memcached`` on top of an in-memory LRU. + +""" + +import logging + +__all__ = ['get_cache'] + +def get_cache(config): + """ A factory that returns a cache object based on the given configuration. + + This function is the only 'public' attribute in this module and is used + to encapsulate the logic behind the cache selection. + + """ + if config.MEMCACHE: + return MemcacheCache(config) + else: + return DefaultCache(config) + +class DefaultCache(object): + """ A simple in process-memory cache using a LRU policy. + + Note: This default cache comes from extracting code logic found in + :module: `rdbms_common`. + + """ + + def __init__(self, config): + # keep a cache of the N most recently retrieved rows of any kind + self._cache = {} + self._lru = LRU(config.RDBMS_CACHE_SIZE) + + + def set(self, key, object): + """ Add the object to the cache. + + Adding an object is a three steps process: + + #. add object to cache + #. add object at start of LRU + #. remove old element if cache has reached max size + + """ + updating = self._cache.has_key(key) + + self._cache[key] = object + + if not updating: + # update the LRU + too_old = self._lru.insert(key) + + if too_old is not None: + del self._cache[too_old] + else: + self._lru.update(key) + + + def get(self, key): + object = self._cache.get(key) + if object is not None: + # push us back to the top of the LRU + self._lru.update(key) + + return object + + def has_key(self, key): + return self._cache.has_key(key) + + + def clear(self, key): + try: + del self._cache[key] + self._lru.remove(key) + except KeyError: + pass + + + def clear_all(self): + self._cache = {} + self._lru.clear_all() + + + def __len__(self): + return len(self._cache) + + + def close(self): + """Close connection to the cache. + + Noop in the case of default cache. + + """ + pass + + +class MemcacheCache(DefaultCache): + """ Memcached backed dict. + + This cache extends the default cache so as to use the process memory as a + first level cache, hitting memcache only when that internal memory misses. + + Objects that are saved in this cache **must** be pickable. + + """ + logger = logging.getLogger("hyperdb") + + def __init__(self, config): + import memcache + + super(MemcacheCache, self).__init__(config) + + self._client = memcache.Client([config.MEMCACHE]) + + self._prefix = "roundup:sql:%s" % config.RDBMS_NAME + + self.logger.debug("Using memcache: %r" % config.MEMCACHE) + + + def _get_key(self, key): + """ Returns a memcache friendly version of the key.""" + # memcache won't accept unicode keys, only string + if isinstance(key, unicode): + key = key.encode("utf-8") + + if self._prefix: + return "%s:%s" % (self._prefix, key) + else: + return key + + def set(self, key, object): + super(MemcacheCache, self).set(key, object) + + self.logger.debug("memcache setting %r to %r" % (key, object)) + self._client.set(self._get_key(key), object) + + + def get(self, key): + # check in memory cache + object = super(MemcacheCache, self).get(key) + if object is not None: + return object + + # check memcache + object = self._client.get(self._get_key(key)) + if object is not None: + self.logger.debug("memcache cache hit for %r" % (key,)) + + # feedback to in memory cache + super(MemcacheCache, self).set(key, object) + + return object + + self.logger.debug("memcache cache miss for %r" % (key,)) + + + def has_key(self, key): + return ( + super(MemcacheCache, self).has_key(key) + or + self._client.get(self._get_key(key)) is not None # memcache has no + # 'has_key' so we + # need to get and + # see if we got + # something + ) + + + def clear(self, key): + super(MemcacheCache, self).clear(key) + + self._client.delete(self._get_key(key)) + + + def clear_all(self): + """ Clears the entire cache. + + Uses the in-memory list of keys to clear the memcache objects. + + """ + logging.getLogger("hyperdb").debug("Clearing caches") + + # we need to work off a copy of the list of keys because we will be + # calling ``self.clear`` which will modify `self._cache` + for key in self._cache.keys()[:]: + logging.getLogger("hyperdb").debug("memcache removing %r" % (key,)) + self.clear(key) + + + def close(self): + """Close connection to the cache. + """ + logging.getLogger().debug("Closing connection to memcache") + self._client.disconnect_all() + + +class LRU(object): + """ A simple implementation of an LRU backed by a list.""" + + def __init__(self, size): + self._size = size + self._lru = [] + + + def insert(self, key): + """ Adds a new key to the begining of the LRU. + + If the key was already present, it is moved to the beginning of the LRU + so the size is not changed. + + If the LRU has reached its max. size, the method removes and returns + the Least Recently Used element. + + :return: ``None`` or the Last Recently Used key if the max size has + been reached. + + """ + if key in self._lru: + self.update(key) + else: + self._lru.insert(0, key) + + if len(self._lru) > self._size: + return self._lru.pop() + + + def update(self, key): + """ Pulls an existing entry at the begining of the LRU.""" + self._lru.remove(key) + self._lru.insert(0, key) + + + def remove(self, key): + self._lru.remove(key) + + + def clear_all(self): + self._lru = [] + + + def __len__(self): + return len(self._lru) --- test_cache.py +++ test_cache.py @@ -0,0 +1,361 @@ +import unittest + +from mock import Mock, patch + +from roundup.backends import cache + + +KEY = "some_key" +OBJECT = {'attr1': 1, 'attr2': 2} + +class ConfigStub(object): + + def __init__(self): + self.RDBMS_CACHE_SIZE = 100 + self.RDBMS_NAME = "some_rdbms" + self.MEMCACHE = "127.0.0.1:11211" + + +class LRUTestCase(unittest.TestCase): + + def setUp(self): + self.size = 4 + self.lru = cache.LRU(self.size) + + self.key_tpl = "key%d" + + + def test_inserting_less_than_size_times_keeps_inserting(self): + n = self.size / 2 + + # Exercise + removed = self._insert_n_elements(n) + + # Verify + self.assertEquals([], removed) + self.assertEquals(n, len(self.lru)) + + + def test_inserting_size_times_keeps_inserting(self): + # Exercise + removed = self._insert_n_elements(self.size) + + # Verify + self.assertEquals([], removed) + self.assertEquals(self.size, self.size) + + self.assertEquals(self.size, len(self.lru)) + + + def test_inserting_more_than_size_times_removes_oldest(self): + self._insert_n_elements(self.size) + + # Exercise/Verify + self.assertEqual(self.size, len(self.lru)) + self.assertEquals(self.key_tpl % 0, self.lru.insert("OneObjectTooMany")) + + self.assertEqual(self.size, len(self.lru)) + self.assertEquals(self.key_tpl % 1, self.lru.insert("EvenOneMore")) + + + def test_inserting_key_twice_updates(self): + key = "SomeKey" + self.lru.update = Mock() + + # Exercise + self.lru.insert(key) + self.lru.insert(key) + + # Verify + self.assertTrue(self.lru.update.called) + + + def test_updating_does_not_change_size(self): + self._insert_n_elements(2) + + len_before = len(self.lru) + + self.lru.update(self.key_tpl % 1) + + self.assertEqual(len_before, len(self.lru)) + + + def test_updating_puts_key_at_begining(self): + key = "KeyToUpdate" + self.lru.insert(key) + + self._insert_n_elements(self.size - 1) + + # Exercise + self.lru.update(key) + + # Verify + # key should only be removed if we perform self.size more insert + removed_list = self._insert_n_elements(self.size - 1, "NewKey%d") + self.assertFalse(key in removed_list) + + self.assertEqual(key, self.lru.insert("LastKey")) + + + def test_clear_all_empties_lru(self): + self._insert_n_elements(self.size) + + # Exercise + self.lru.clear_all() + + # Verify + self.assertEqual(0 , len(self.lru)) + + + # /////////////////////////// + # Helper methods + # /////////////////////////// + def _insert_n_elements(self, n, key_tpl=None): + if key_tpl == None: + key_tpl = self.key_tpl + + removed_list = [] + for i in range(n): + removed = self.lru.insert(key_tpl % i) + if removed is not None: + removed_list.append(removed) + + return removed_list + + +class DefaultCacheTestCase(unittest.TestCase): + + @patch('roundup.backends.cache.LRU', spec=True) + def setUp(self, mock): + self.cache = cache.DefaultCache(ConfigStub()) + + self.mock_lru = self.cache._lru + # make sure insert returns nothing byt default + self.mock_lru.insert.return_value = None + + + def test_new_cache_is_empty(self): + self.assertEquals(0, len(self.cache)) + + + def test_set_saves_into_cache(self): + # Exercise + self.cache.set(KEY, OBJECT) + + # Verify + self.assertEqual(1, len(self.cache)) + + + def test_set_inserts_into_lru(self): + # Exercise + self.cache.set(KEY, OBJECT) + + # Verify + self.mock_lru.insert.assert_called_with(KEY) + + + def test_set_twice_updates_lru(self): + # Exercise + self.cache.set(KEY, OBJECT) + self.cache.set(KEY, OBJECT) + + # Verify + self.mock_lru.update.assert_called_with(KEY) + + + def test_get_before_set_returns_none(self): + cached = self.cache.get(KEY) + + self.assertEquals(None, cached) + + + def test_has_key_before_set_returns_false(self): + has_key = self.cache.has_key("UnknownKey") + + self.assertFalse(has_key) + + + def test_get_after_set_returns_object(self): + self.cache.set(KEY, OBJECT) + cached = self.cache.get(KEY) + + self.assertEquals(OBJECT, cached) + + + def test_has_key_after_set_returns_true(self): + self.cache.set(KEY, OBJECT) + has_key = self.cache.has_key(KEY) + + self.assertTrue(has_key) + + + def test_clear_removes_key(self): + self.cache.set(KEY, OBJECT) + + self.cache.clear(KEY) + + self.assertEqual(None, self.cache.get(KEY)) + self.assertFalse(self.cache.has_key(KEY)) + + + def test_clear_all_removes_all_keys(self): + key_tpl = "SomeKey%d" + object_tpl = "SomeObject%d" + for i in range(ConfigStub().RDBMS_CACHE_SIZE): + self.cache.set(key_tpl % i, object_tpl) + + # Exercise + self.cache.clear_all() + + # Verify + self.assertEquals(0, len(self.cache)) + for i in range(ConfigStub().RDBMS_CACHE_SIZE): + self.assertFalse(self.cache.has_key(key_tpl % i)) + self.assertEquals(None, self.cache.get(key_tpl % i)) + + + def test_close_exists(self): + try: + self.cache.close() + except AttributeError: + self.fail("%s should have a 'close' callable attribute." + % self.cache.__class__) + +class MemcacheCacheTestCase(DefaultCacheTestCase): + """Class for testing `MemcacheCache`. + + Extends `DefaultCacheTestCase` because `MemcacheCache` should use a + `DefaultCache` as first level cache, therefore we want it to have the same + unit tests. + + """ + @patch('memcache.Client', spec=True) + @patch('roundup.backends.cache.LRU', spec=True) + def setUp(self, mock_lru_class, mock_client_class): + self.cache = cache.MemcacheCache(ConfigStub()) + # to ease testing, don't change keys for memcache + self.cache._prefix = "" + + self.mock_lru = self.cache._lru + # Make sure insert returns nothing by default + self.mock_lru.insert.return_value = None + + self.mock_client = self.cache._client + # Make sure the cache is empty by default + self.mock_client.get.return_value = None + + self.mock_client_class = mock_client_class + + + def test_cache_creates_memcache_client_properly(self): + self.mock_client_class.assert_called_with([ConfigStub().MEMCACHE]) + + + def test_set_adds_into_memcache(self): + self.cache.set(KEY, OBJECT) + + self.mock_client.set.called + self.mock_client.set.assert_called_with(KEY, OBJECT) + + + def test_set_does_not_pass_unicode_keys(self): + self.cache.set(unicode(KEY), OBJECT) + + self.mock_client.set.called + self.mock_client.set.assert_called_with(KEY, OBJECT) + + + def test_set_handles_ascii_keys(self): + non_ascii_key = u"\u0252mlauts\u0224\u0233" + + self.cache.set(non_ascii_key, OBJECT) + + # Verify + expected_key = non_ascii_key.encode("utf-8") + self.mock_client.set.called + self.mock_client.set.assert_called_with(expected_key, OBJECT) + + + @patch('roundup.backends.cache.DefaultCache.get') + def test_get_returns_from_default_first(self, mock_get): + mock_get.return_value = OBJECT + + # Exercise + obj = self.cache.get(KEY) + + # Verify + self.assertTrue(mock_get.called) + self.assertEqual(OBJECT, obj) + + + def test_get_returns_from_memcache_if_not_in_default(self): + self.mock_client.get.return_value = OBJECT + + # Exercise + obj = self.cache.get(KEY) + + # Verify + self.assertTrue(self.mock_client.get.called) + self.assertEqual(OBJECT, obj) + + + @patch('roundup.backends.cache.DefaultCache.set') + def test_get_from_memcache_feeds_back_into_default(self, mock_set): + self.mock_client.get.return_value = OBJECT + + # Exercise + self.cache.get(KEY) + + # Verify + mock_set.assert_called_with(KEY, OBJECT) + + + @patch('roundup.backends.cache.DefaultCache.has_key') + def test_has_key_searches_in_default_first(self, mock_has_key): + mock_has_key.return_value = True + + # Exercise + has = self.cache.has_key(KEY) + + # Verify + self.assertTrue(mock_has_key.called) + self.assertTrue(has) + + + def test_has_key_searches_in_memcache_if_not_in_default(self): + # memcache has no 'has_key' + # you need to get and see if you obtained something + self.mock_client.get.return_value = OBJECT + + # Exercise + has = self.cache.has_key(KEY) + + # Verify + self.assertTrue(self.mock_client.get.called) + self.assertTrue(has) + + + @patch('roundup.backends.cache.DefaultCache.clear') + def test_clear_removes_from_default_and_memcache(self, mock_clear): + # Exercise + self.cache.clear(KEY) + + # Verify + mock_clear.assert_called_with(KEY) + self.mock_client.delete.assert_called_with(KEY) + + + def test_close_disconnects_from_memcached_servers(self): + # Exercise + self.cache.close() + + self.assertTrue(self.mock_client.disconnect_all.called) + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(LRUTestCase)) + suite.addTest(unittest.makeSuite(DefaultCacheTestCase)) + suite.addTest(unittest.makeSuite(MemcacheCacheTestCase)) + + +if __name__ == "__main__": + unittest.main()