From fb1926cd6655d5b7462d591ca8c152b5a596c0e6 Mon Sep 17 00:00:00 2001 From: Josh Roesslein Date: Sun, 9 Aug 2009 13:47:13 -0500 Subject: [PATCH] Implemented file-based cache. --- README | 1 + TODO | 3 +- tweepy/cache.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/README b/README index 7e52ba5..6b28470 100644 --- a/README +++ b/README @@ -14,6 +14,7 @@ Features: Supports most of the twitter api endpoints. Python 3 support. (See py3k branch) Streaming API + Cache system (memory and file based) Getting started: Check out the tutorial folder to get started with Tweepy. diff --git a/TODO b/TODO index c178025..b61c1c4 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,7 @@ Stuff that needs to be done... - finish unit tests - search API - caching system - + in-memory cache done. now just need a file-based cache (pickle?) + - memcache? database? + + memory and file caches implemented - needs docs, tutors, examples, etc - commandline client diff --git a/tweepy/cache.py b/tweepy/cache.py index 9062f22..3cb91bb 100644 --- a/tweepy/cache.py +++ b/tweepy/cache.py @@ -4,6 +4,10 @@ import time import threading +import os +import hashlib +import fcntl +import cPickle as pickle """Cache interface""" class Cache(object): @@ -71,10 +75,7 @@ class MemoryCache(Cache): # use provided timeout in arguments if provided # otherwise use the one provided during init. - if timeout is None: - _timeout = self.timeout - else: - _timeout = timeout + _timeout = self.timeout if timeout is None else timeout # make sure entry is not expired if self._is_expired(entry, _timeout): @@ -95,3 +96,89 @@ class MemoryCache(Cache): with self.lock: self._entries.clear() +"""File-based cache""" +class FileCache(Cache): + + def __init__(self, cache_dir, timeout=60): + Cache.__init__(self, timeout) + if os.path.exists(cache_dir) is False: + os.mkdir(cache_dir) + self.cache_dir = cache_dir + self.lock = threading.Lock() + + def _get_path(self, key): + md5 = hashlib.md5() + md5.update(key) + return os.path.join(self.cache_dir, md5.hexdigest()) + + def _lock_file(self, path, exclusive=True): + lock_path = path + '.lock' + if exclusive is True: + f_lock = open(lock_path, 'w') + fcntl.lockf(f_lock, fcntl.LOCK_EX) + else: + f_lock = open(lock_path, 'r') + fcntl.lockf(f_lock, fcntl.LOCK_SH) + if os.path.exists(lock_path) is False: + f_lock.close() + return None + return f_lock + + def _delete_file(self, path): + os.remove(path) + os.remove(path + '.lock') + + def store(self, key, value): + path = self._get_path(key) + with self.lock: + # acquire lock and open file + f_lock = self._lock_file(path) + datafile = open(path, 'wb') + + # write data + pickle.dump((time.time(), value), datafile) + + # close and unlock file + datafile.close() + f_lock.close() + + def get(self, key, timeout=None): + return self._get(self._get_path(key), timeout) + + def _get(self, path, timeout): + if os.path.exists(path) is False: + # no record + return None + while self.lock: + # acquire lock and open + f_lock = self._lock_file(path, False) + if f_lock is None: + # does not exist + return None + datafile = open(path, 'rb') + + # read pickled object + created_time, value = pickle.load(datafile) + datafile.close() + + # check if value is expired + _timeout = self.timeout if timeout is None else timeout + if _timeout > 0 and (time.time() - created_time) >= _timeout: + # expired! delete from cache + value = None + self._delete_file(path) + + # unlock and return result + f_lock.close() + return value + + def cleanup(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): continue + self._get(os.path.join(self.cache_dir, entry), None) + + def flush(self): + for entry in os.listdir(self.cache_dir): + if entry.endswith('.lock'): continue + self._delete_file(os.path.join(self.cache_dir, entry)) + -- 2.25.1