Source code for wheezy.caching.memory

""" ``memory`` module.
"""

from time import time as unixtime

from wheezy.caching.comp import allocate_lock
from wheezy.caching.comp import iteritems
from wheezy.caching.comp import xrange


def expires(now, time):
[docs] """ ``time`` is below 1 month >>> expires(10, 1) 11 more than month >>> expires(10, 3000000) 3000000 otherwise >>> expires(0, 0) 2147483647 >>> expires(0, -1) 2147483647 """ if time > 0: if time < 2592000: # 1 month return now + time else: return time else: return 0x7FFFFFFF def find_expired(bucket_items, now):
[docs] """ If there are no expired items in the bucket returns empty list >>> bucket_items = [('k1', 1), ('k2', 2), ('k3', 3)] >>> find_expired(bucket_items, 0) [] >>> bucket_items [('k1', 1), ('k2', 2), ('k3', 3)] Expired items are returned in the list and deleted from the bucket >>> find_expired(bucket_items, 2) ['k1'] >>> bucket_items [('k2', 2), ('k3', 3)] """ expired_keys = [] for i in xrange(len(bucket_items) - 1, -1, -1): key, expires = bucket_items[i] if expires < now: expired_keys.append(key) del bucket_items[i] return expired_keys class CacheItem(object):
[docs] """ A single cache item stored in cache. """ __slots__ = ('key', 'value', 'expires') def __init__(self, key, value, expires): self.key = key self.value = value self.expires = expires class MemoryCache(object):
[docs] """ Effectively implements in-memory cache. """ def __init__(self, buckets=60, bucket_interval=15): self.period = buckets * bucket_interval self.interval = bucket_interval self.items = {} self.lock = allocate_lock() self.expire_buckets = [ (allocate_lock(), []) for i in xrange(0, buckets)] self.last_expire_bucket_id = -1 def set(self, key, value, time=0, namespace=None):
[docs] """ Sets a key's value, regardless of previous contents in cache. >>> c = MemoryCache() >>> c.set('k', 'v', 100) True """ return self.store(key, value, time, 0) def set_multi(self, mapping, time=0, namespace=None):
[docs] """ Set multiple keys' values at once. >>> c = MemoryCache() >>> c.set_multi({'k1': 1, 'k2': 2}, 100) [] """ return self.store_multi(mapping, time) def add(self, key, value, time=0, namespace=None):
[docs] """ Sets a key's value, if and only if the item is not already. >>> c = MemoryCache() >>> c.add('k', 'v', 100) True >>> c.add('k', 'v', 100) False """ return self.store(key, value, time, 1) def add_multi(self, mapping, time=0, namespace=None):
[docs] """ Adds multiple values at once, with no effect for keys already in cache. >>> c = MemoryCache() >>> c.add_multi({'k': 'v'}, 100) [] >>> c.add_multi({'k': 'v'}, 100) ['k'] """ return self.store_multi(mapping, time, 1) def replace(self, key, value, time=0, namespace=None):
[docs] """ Replaces a key's value, failing if item isn't already. >>> c = MemoryCache() >>> c.replace('k', 'v', 100) False >>> c.add('k', 'v', 100) True >>> c.replace('k', 'v', 100) True """ return self.store(key, value, time, 2) def replace_multi(self, mapping, time=0, namespace=None):
[docs] """ Replaces multiple values at once, with no effect for keys not in cache. >>> c = MemoryCache() >>> c.replace_multi({'k': 'v'}, 100) ['k'] >>> c.add_multi({'k': 'v'}, 100) [] >>> c.replace_multi({'k': 'v'}, 100) [] """ return self.store_multi(mapping, time, 2) def get(self, key, namespace=None):
[docs] """ Looks up a single key. If ``key`` is not found return None >>> c = MemoryCache() >>> c.get('k') Otherwise return value >>> c.set('k', 'v', 100) True >>> c.get('k') 'v' There is item in cached that expired >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.get('k') """ now = int(unixtime()) items = self.items self.lock.acquire(1) try: try: entry = items[key] if entry.expires < now: del items[key] return None return entry.value except KeyError: return None finally: self.lock.release() def get_multi(self, keys, namespace=None):
[docs] """ Looks up multiple keys from cache in one operation. This is the recommended way to do bulk loads. >>> c = MemoryCache() >>> c.get_multi(('k1', 'k2', 'k3')) {} >>> c.store('k1', 'v1', 100) True >>> c.store('k2', 'v2', 100) True >>> sorted(c.get_multi(('k1', 'k2')).items()) [('k1', 'v1'), ('k2', 'v2')] There is item in cache that expired >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.get_multi(('k', )) {} """ now = int(unixtime()) results = {} items = self.items self.lock.acquire(1) try: for key in keys: try: entry = items[key] if entry.expires < now: del items[key] else: results[key] = entry.value except KeyError: pass finally: self.lock.release() return results def delete(self, key, seconds=0, namespace=None):
[docs] """ Deletes a key from cache. If ``key`` is not found return False >>> c = MemoryCache() >>> c.delete('k') False >>> c.store('k', 'v', 100) True >>> c.delete('k') True There is item in cache that expired >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.delete('k') False """ now = int(unixtime()) items = self.items self.lock.acquire(1) try: try: entry = items[key] del items[key] if entry.expires < now: return False return True except KeyError: return False finally: self.lock.release() def delete_multi(self, keys, seconds=0, namespace=None):
[docs] """ Delete multiple keys at once. >>> c = MemoryCache() >>> c.delete_multi(('k1', 'k2', 'k3')) True >>> c.store_multi({'k1':1, 'k2': 2}, 100) [] >>> c.delete_multi(('k1', 'k2')) True There is item in cached that expired >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.get_multi(('k', )) {} """ items = self.items self.lock.acquire(1) try: for key in keys: try: del items[key] except KeyError: pass finally: self.lock.release() return True def incr(self, key, delta=1, namespace=None, initial_value=None):
[docs] """ Atomically increments a key's value. The value, if too large, will wrap around. If the key does not yet exist in the cache and you specify an initial_value, the key's value will be set to this initial value and then incremented. If the key does not exist and no initial_value is specified, the key's value will not be set. >>> c = MemoryCache() >>> c.incr('k') >>> c.incr('k', initial_value=0) 1 >>> c.incr('k') 2 There is item in cached that expired >>> c.items['k'] = CacheItem('k', 1, 1) >>> c.incr('k') """ now = int(unixtime()) items = self.items self.lock.acquire(1) try: try: entry = items[key] if entry.expires < now: del items[key] entry = None except KeyError: entry = None if entry is None: if initial_value is None: return None else: entry = items[key] = CacheItem( key, initial_value, expires(now, 0)) value = entry.value = entry.value + delta return value finally: self.lock.release() def decr(self, key, delta=1, namespace=None, initial_value=None):
[docs] """ Atomically decrements a key's value. The value, if too large, will wrap around. If the key does not yet exist in the cache and you specify an initial_value, the key's value will be set to this initial value and then decremented. If the key does not exist and no initial_value is specified, the key's value will not be set. >>> c = MemoryCache() >>> c.decr('k') >>> c.decr('k', initial_value=10) 9 >>> c.decr('k') 8 """ return self.incr(key, -delta, namespace, initial_value) def store(self, key, value, time=0, op=0):
[docs] """ There is item in cached that expired >>> c = MemoryCache() >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.store('k', 'v', 100) True There is item in expire_buckets that expired >>> c = MemoryCache() >>> i = int((int(unixtime()) % c.period) ... / c.interval) - 1 >>> c.expire_buckets[i] = (allocate_lock(), [('x', 10)]) >>> c.store('k', 'v', 100) True """ now = int(unixtime()) time = expires(now, time) items = self.items self.lock.acquire(1) try: try: entry = items[key] if entry.expires < now: del items[key] elif op == 1: # add return False except KeyError: if op == 2: # replace return False items[key] = CacheItem(key, value, time) finally: self.lock.release() if time < 0x7FFFFFFF: expired_keys = None bucket_id = int((now % self.period) / self.interval) bucket_lock, bucket_items = self.expire_buckets[bucket_id - 1] bucket_lock.acquire(1) try: if self.last_expire_bucket_id != bucket_id: self.last_expire_bucket_id = bucket_id expired_keys = find_expired(bucket_items, now) bucket_items.append((key, time)) finally: bucket_lock.release() if expired_keys: self.delete_multi(expired_keys) return True def store_multi(self, mapping, time=0, op=0):
[docs] """ There is item in cached that expired >>> c = MemoryCache() >>> c.items['k'] = CacheItem('k', 'v', 1) >>> c.store_multi({'k': 'v'}, 100) [] There is item in expire_buckets that expired >>> c = MemoryCache() >>> i = int((int(unixtime()) % c.period) ... / c.interval) - 1 >>> c.expire_buckets[i] = (allocate_lock(), [('x', 10)]) >>> c.store_multi({'k': 'v'}, 100) [] """ now = int(unixtime()) time = expires(now, time) items = self.items keys_failed = [] succeeded = [] self.lock.acquire(1) try: for key, value in iteritems(mapping): try: entry = items[key] if entry.expires < now: del items[key] elif op == 1: # add keys_failed.append(key) continue except KeyError: if op == 2: # replace keys_failed.append(key) continue items[key] = CacheItem(key, value, time) succeeded.append((key, time)) finally: self.lock.release() if time < 0x7FFFFFFF and succeeded: expired_keys = None bucket_id = int((now % self.period) / self.interval) bucket_lock, bucket_items = self.expire_buckets[bucket_id - 1] bucket_lock.acquire(1) try: if self.last_expire_bucket_id != bucket_id: self.last_expire_bucket_id = bucket_id expired_keys = find_expired(bucket_items, now) bucket_items.extend(succeeded) finally: bucket_lock.release() if expired_keys: self.delete_multi(expired_keys) return keys_failed def flush_all(self):
[docs] """ Deletes everything in cache. >>> c = MemoryCache() >>> c.set_multi({'k1': 1, 'k2': 2}, 100) [] >>> c.flush_all() True """ self.lock.acquire(1) try: self.items.clear() for bucket_lock, bucket_items in self.expire_buckets: bucket_lock.acquire(1) try: del bucket_items[:] finally: bucket_lock.release() finally: self.lock.release() return True if __name__ == '__main__': # pragma: nocover
import doctest doctest.testmod()