Package web2py :: Package gluon :: Module cache
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.cache

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8   
  9  Basic caching classes and methods 
 10  ================================= 
 11   
 12  - Cache - The generic caching object interfacing with the others 
 13  - CacheInRam - providing caching in ram 
 14  - CacheInDisk - provides caches on disk 
 15   
 16  Memcache is also available via a different module (see gluon.contrib.memcache) 
 17   
 18  When web2py is running on Google App Engine, 
 19  caching will be provided by the GAE memcache 
 20  (see gluon.contrib.gae_memcache) 
 21  """ 
 22   
 23  import time 
 24  import portalocker 
 25  import shelve 
 26  import thread 
 27  import os 
 28  import logging 
 29  import re 
 30   
 31  logger = logging.getLogger("web2py.cache") 
 32   
 33  __all__ = ['Cache'] 
 34   
 35   
 36  DEFAULT_TIME_EXPIRE = 300 
 37   
 38   
39 -class CacheAbstract(object):
40 """ 41 Abstract class for cache implementations. 42 Main function is now to provide referenced api documentation. 43 44 Use CacheInRam or CacheOnDisk instead which are derived from this class. 45 """ 46 47 cache_stats_name = 'web2py_cache_statistics' 48
49 - def __init__(self, request=None):
50 """ 51 Paremeters 52 ---------- 53 request: 54 the global request object 55 """ 56 raise NotImplementedError
57
58 - def __call__(self, key, f, 59 time_expire = DEFAULT_TIME_EXPIRE):
60 """ 61 Tries retrieve the value corresponding to `key` from the cache of the 62 object exists and if it did not expire, else it called the function `f` 63 and stores the output in the cache corresponding to `key`. In the case 64 the output of the function is returned. 65 66 :param key: the key of the object to be store or retrieved 67 :param f: the function, whose output is to be cached 68 :param time_expire: expiration of the cache in microseconds 69 70 - `time_expire` is used to compare the current time with the time when 71 the requested object was last saved in cache. It does not affect 72 future requests. 73 - Setting `time_expire` to 0 or negative value forces the cache to 74 refresh. 75 76 If the function `f` is `None` the cache is cleared. 77 """ 78 raise NotImplementedError
79
80 - def clear(self, regex=None):
81 """ 82 Clears the cache of all keys that match the provided regular expression. 83 If no regular expression is provided, it clears all entries in cache. 84 85 Parameters 86 ---------- 87 regex: 88 if provided, only keys matching the regex will be cleared. 89 Otherwise all keys are cleared. 90 """ 91 92 raise NotImplementedError
93
94 - def increment(self, key, value=1):
95 """ 96 Increments the cached value for the given key by the amount in value 97 98 Parameters 99 ---------- 100 key: 101 key for the cached object to be incremeneted 102 value: 103 amount of the increment (defaults to 1, can be negative) 104 """ 105 raise NotImplementedError
106
107 - def _clear(self, storage, regex):
108 """ 109 Auxiliary function called by `clear` to search and clear cache entries 110 """ 111 r = re.compile(regex) 112 for (key, value) in storage.items(): 113 if r.match(str(key)): 114 del storage[key]
115
116 -class CacheInRam(CacheAbstract):
117 """ 118 Ram based caching 119 120 This is implemented as global (per process, shared by all threads) 121 dictionary. 122 A mutex-lock mechanism avoid conflicts. 123 """ 124 125 locker = thread.allocate_lock() 126 meta_storage = {} 127
128 - def __init__(self, request=None):
129 self.locker.acquire() 130 self.request = request 131 if request: 132 app = request.application 133 else: 134 app = '' 135 if not app in self.meta_storage: 136 self.storage = self.meta_storage[app] = {CacheAbstract.cache_stats_name: { 137 'hit_total': 0, 138 'misses': 0, 139 }} 140 else: 141 self.storage = self.meta_storage[app] 142 self.locker.release()
143
144 - def clear(self, regex=None):
145 self.locker.acquire() 146 storage = self.storage 147 if regex == None: 148 storage.clear() 149 else: 150 self._clear(storage, regex) 151 152 if not CacheAbstract.cache_stats_name in storage.keys(): 153 storage[CacheAbstract.cache_stats_name] = { 154 'hit_total': 0, 155 'misses': 0, 156 } 157 158 self.locker.release()
159
160 - def __call__(self, key, f, 161 time_expire = DEFAULT_TIME_EXPIRE):
162 """ 163 Attention! cache.ram does not copy the cached object. It just stores a reference to it. 164 Turns out the deepcopying the object has some problems: 165 1) would break backward compatibility 166 2) would be limiting because people may want to cache live objects 167 3) would work unless we deepcopy no storage and retrival which would make things slow. 168 Anyway. You can deepcopy explicitly in the function generating the value to be cached. 169 """ 170 171 dt = time_expire 172 173 self.locker.acquire() 174 item = self.storage.get(key, None) 175 if item and f == None: 176 del self.storage[key] 177 self.storage[CacheAbstract.cache_stats_name]['hit_total'] += 1 178 self.locker.release() 179 180 if f is None: 181 return None 182 if item and (dt == None or item[0] > time.time() - dt): 183 return item[1] 184 value = f() 185 186 self.locker.acquire() 187 self.storage[key] = (time.time(), value) 188 self.storage[CacheAbstract.cache_stats_name]['misses'] += 1 189 self.locker.release() 190 return value
191
192 - def increment(self, key, value=1):
193 self.locker.acquire() 194 try: 195 if key in self.storage: 196 value = self.storage[key][1] + value 197 self.storage[key] = (time.time(), value) 198 except BaseException, e: 199 self.locker.release() 200 raise e 201 self.locker.release() 202 return value
203 204
205 -class CacheOnDisk(CacheAbstract):
206 """ 207 Disk based cache 208 209 This is implemented as a shelve object and it is shared by multiple web2py 210 processes (and threads) as long as they share the same filesystem. 211 The file is locked wen accessed. 212 213 Disk cache provides persistance when web2py is started/stopped but it slower 214 than `CacheInRam` 215 216 Values stored in disk cache must be pickable. 217 """ 218
219 - def __init__(self, request, folder=None):
220 self.request = request 221 222 # Lets test if the cache folder exists, if not 223 # we are going to create it 224 folder = folder or os.path.join(request.folder, 'cache') 225 226 if not os.path.exists(folder): 227 os.mkdir(folder) 228 229 ### we need this because of a possible bug in shelve that may 230 ### or may not lock 231 self.locker_name = os.path.join(request.folder, 232 'cache/cache.lock') 233 self.shelve_name = os.path.join(request.folder, 234 'cache/cache.shelve') 235 236 locker, locker_locked = None, False 237 try: 238 locker = open(self.locker_name, 'a') 239 portalocker.lock(locker, portalocker.LOCK_EX) 240 locker_locked = True 241 storage = shelve.open(self.shelve_name) 242 243 if not storage.has_key(CacheAbstract.cache_stats_name): 244 storage[CacheAbstract.cache_stats_name] = { 245 'hit_total': 0, 246 'misses': 0, 247 } 248 storage.sync() 249 except ImportError: 250 pass # no module _bsddb, ignoring exception now so it makes a ticket only if used 251 except: 252 logger.error('corrupted file: %s' % self.shelve_name) 253 if locker_locked: 254 portalocker.unlock(locker) 255 if locker: 256 locker.close()
257
258 - def clear(self, regex=None):
259 locker = open(self.locker_name,'a') 260 portalocker.lock(locker, portalocker.LOCK_EX) 261 storage = shelve.open(self.shelve_name) 262 if regex == None: 263 storage.clear() 264 else: 265 self._clear(storage, regex) 266 if not CacheAbstract.cache_stats_name in storage.keys(): 267 storage[CacheAbstract.cache_stats_name] = { 268 'hit_total': 0, 269 'misses': 0, 270 } 271 storage.sync() 272 portalocker.unlock(locker) 273 locker.close()
274
275 - def __call__(self, key, f, 276 time_expire = DEFAULT_TIME_EXPIRE):
277 dt = time_expire 278 279 locker = open(self.locker_name,'a') 280 portalocker.lock(locker, portalocker.LOCK_EX) 281 282 storage = shelve.open(self.shelve_name) 283 284 item = storage.get(key, None) 285 if item and f == None: 286 del storage[key] 287 288 storage[CacheAbstract.cache_stats_name] = { 289 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'] + 1, 290 'misses': storage[CacheAbstract.cache_stats_name]['misses'] 291 } 292 293 storage.sync() 294 295 portalocker.unlock(locker) 296 locker.close() 297 298 if f is None: 299 return None 300 if item and (dt == None or item[0] > time.time() - dt): 301 return item[1] 302 value = f() 303 304 locker = open(self.locker_name,'a') 305 portalocker.lock(locker, portalocker.LOCK_EX) 306 307 storage[key] = (time.time(), value) 308 309 storage[CacheAbstract.cache_stats_name] = { 310 'hit_total': storage[CacheAbstract.cache_stats_name]['hit_total'], 311 'misses': storage[CacheAbstract.cache_stats_name]['misses'] + 1 312 } 313 314 storage.sync() 315 316 portalocker.unlock(locker) 317 locker.close() 318 319 return value
320
321 - def increment(self, key, value=1):
322 locker = open(self.locker_name,'a') 323 portalocker.lock(locker, portalocker.LOCK_EX) 324 storage = shelve.open(self.shelve_name) 325 try: 326 if key in storage: 327 value = storage[key][1] + value 328 storage[key] = (time.time(), value) 329 storage.sync() 330 except BaseException, e: 331 portalocker.unlock(locker) 332 locker.close() 333 raise e 334 portalocker.unlock(locker) 335 locker.close() 336 return value
337 338
339 -class Cache(object):
340 """ 341 Sets up generic caching, creating an instance of both CacheInRam and 342 CacheOnDisk. 343 In case of GAE will make use of gluon.contrib.gae_memcache. 344 345 - self.ram is an instance of CacheInRam 346 - self.disk is an instance of CacheOnDisk 347 """ 348
349 - def __init__(self, request):
350 """ 351 Parameters 352 ---------- 353 request: 354 the global request object 355 """ 356 # GAE will have a special caching 357 import settings 358 if settings.global_settings.web2py_runtime_gae: 359 from contrib.gae_memcache import MemcacheClient 360 self.ram=self.disk=MemcacheClient(request) 361 else: 362 # Otherwise use ram (and try also disk) 363 self.ram = CacheInRam(request) 364 try: 365 self.disk = CacheOnDisk(request) 366 except IOError: 367 logger.warning('no cache.disk (IOError)') 368 except AttributeError: 369 # normally not expected anymore, as GAE has already 370 # been accounted for 371 logger.warning('no cache.disk (AttributeError)')
372
373 - def __call__(self, 374 key = None, 375 time_expire = DEFAULT_TIME_EXPIRE, 376 cache_model = None):
377 """ 378 Decorator function that can be used to cache any function/method. 379 380 Example:: 381 382 @cache('key', 5000, cache.ram) 383 def f(): 384 return time.ctime() 385 386 When the function f is called, web2py tries to retrieve 387 the value corresponding to `key` from the cache of the 388 object exists and if it did not expire, else it calles the function `f` 389 and stores the output in the cache corresponding to `key`. In the case 390 the output of the function is returned. 391 392 :param key: the key of the object to be store or retrieved 393 :param time_expire: expiration of the cache in microseconds 394 :param cache_model: `cache.ram`, `cache.disk`, or other 395 (like `cache.memcache` if defined). It defaults to `cache.ram`. 396 397 Notes 398 ----- 399 `time_expire` is used to compare the curret time with the time when the 400 requested object was last saved in cache. It does not affect future 401 requests. 402 Setting `time_expire` to 0 or negative value forces the cache to 403 refresh. 404 405 If the function `f` is an action, we suggest using 406 `request.env.path_info` as key. 407 """ 408 if not cache_model: 409 cache_model = self.ram 410 411 def tmp(func): 412 return lambda: cache_model(key, func, time_expire)
413 414 return tmp
415