From 13a0fcb79dec2ba38f054bada53b5b8629ea7439 Mon Sep 17 00:00:00 2001 From: Joshua Date: Thu, 28 Jan 2010 23:47:39 -0600 Subject: [PATCH] Refactored the way tweepy does parsing to make it more customizable by developers. All parsing of the response payload is now handled by a Parser class defined in tweepy/parsers.py The default parser used is ModelParser which parses a JSON payload into a model instance. Developers may define and use their own custom parsers by extending the Parser class. To use the custom parser: api = API(parser=MyParser()) --- tweepy/api.py | 158 +++++++++++++------------ tweepy/binder.py | 46 ++------ tweepy/models.py | 179 ++++++++++++++++++++++++++++- tweepy/parsers.py | 274 ++++---------------------------------------- tweepy/streaming.py | 4 +- tweepy/utils.py | 71 ++++++++++++ 6 files changed, 359 insertions(+), 373 deletions(-) create mode 100644 tweepy/utils.py diff --git a/tweepy/api.py b/tweepy/api.py index 4dfdafb..b20e343 100644 --- a/tweepy/api.py +++ b/tweepy/api.py @@ -7,8 +7,7 @@ import mimetypes from tweepy.binder import bind_api from tweepy.error import TweepError -from tweepy.parsers import * -from tweepy.models import ModelFactory +from tweepy.parsers import ModelParser class API(object): @@ -18,7 +17,7 @@ class API(object): host='api.twitter.com', search_host='search.twitter.com', cache=None, secure=False, api_root='/1', search_root='', retry_count=0, retry_delay=0, retry_errors=None, - model_factory=None): + parser=None): self.auth = auth_handler self.host = host self.search_host = search_host @@ -29,19 +28,19 @@ class API(object): self.retry_count = retry_count self.retry_delay = retry_delay self.retry_errors = retry_errors - self.model_factory = model_factory or ModelFactory + self.parser = parser or ModelParser() """ statuses/public_timeline """ public_timeline = bind_api( path = '/statuses/public_timeline.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = [] ) """ statuses/home_timeline """ home_timeline = bind_api( path = '/statuses/home_timeline.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -49,7 +48,7 @@ class API(object): """ statuses/friends_timeline """ friends_timeline = bind_api( path = '/statuses/friends_timeline.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -57,7 +56,7 @@ class API(object): """ statuses/user_timeline """ user_timeline = bind_api( path = '/statuses/user_timeline.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['id', 'user_id', 'screen_name', 'since_id', 'max_id', 'count', 'page'] ) @@ -65,7 +64,7 @@ class API(object): """ statuses/mentions """ mentions = bind_api( path = '/statuses/mentions.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -73,7 +72,7 @@ class API(object): """ statuses/retweeted_by_me """ retweeted_by_me = bind_api( path = '/statuses/retweeted_by_me.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -81,7 +80,7 @@ class API(object): """ statuses/retweeted_to_me """ retweeted_to_me = bind_api( path = '/statuses/retweeted_to_me.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -89,7 +88,7 @@ class API(object): """ statuses/retweets_of_me """ retweets_of_me = bind_api( path = '/statuses/retweets_of_me.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -97,7 +96,7 @@ class API(object): """ statuses/show """ get_status = bind_api( path = '/statuses/show.json', - parser = parse_status, + payload_type = 'status', allowed_param = ['id'] ) @@ -105,7 +104,7 @@ class API(object): update_status = bind_api( path = '/statuses/update.json', method = 'POST', - parser = parse_status, + payload_type = 'status', allowed_param = ['status', 'in_reply_to_status_id', 'lat', 'long', 'source'], require_auth = True ) @@ -114,7 +113,7 @@ class API(object): destroy_status = bind_api( path = '/statuses/destroy.json', method = 'DELETE', - parser = parse_status, + payload_type = 'status', allowed_param = ['id'], require_auth = True ) @@ -123,7 +122,7 @@ class API(object): retweet = bind_api( path = '/statuses/retweet/{id}.json', method = 'POST', - parser = parse_status, + payload_type = 'status', allowed_param = ['id'], require_auth = True ) @@ -131,7 +130,7 @@ class API(object): """ statuses/retweets """ retweets = bind_api( path = '/statuses/retweets/{id}.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['id', 'count'], require_auth = True ) @@ -139,7 +138,7 @@ class API(object): """ users/show """ get_user = bind_api( path = '/users/show.json', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'] ) @@ -150,7 +149,7 @@ class API(object): """ users/search """ search_users = bind_api( path = '/users/search.json', - parser = parse_users, + payload_type = 'user', payload_list = True, require_auth = True, allowed_param = ['q', 'per_page', 'page'] ) @@ -158,21 +157,21 @@ class API(object): """ statuses/friends """ friends = bind_api( path = '/statuses/friends.json', - parser = parse_users, + payload_type = 'user', payload_list = True, allowed_param = ['id', 'user_id', 'screen_name', 'page', 'cursor'] ) """ statuses/followers """ followers = bind_api( path = '/statuses/followers.json', - parser = parse_users, + payload_type = 'user', payload_list = True, allowed_param = ['id', 'user_id', 'screen_name', 'page', 'cursor'] ) """ direct_messages """ direct_messages = bind_api( path = '/direct_messages.json', - parser = parse_directmessages, + payload_type = 'direct_message', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -180,7 +179,7 @@ class API(object): """ direct_messages/sent """ sent_direct_messages = bind_api( path = '/direct_messages/sent.json', - parser = parse_directmessages, + payload_type = 'direct_message', payload_list = True, allowed_param = ['since_id', 'max_id', 'count', 'page'], require_auth = True ) @@ -189,7 +188,7 @@ class API(object): send_direct_message = bind_api( path = '/direct_messages/new.json', method = 'POST', - parser = parse_dm, + payload_type = 'direct_message', allowed_param = ['user', 'screen_name', 'user_id', 'text'], require_auth = True ) @@ -198,7 +197,7 @@ class API(object): destroy_direct_message = bind_api( path = '/direct_messages/destroy.json', method = 'DELETE', - parser = parse_dm, + payload_type = 'direct_message', allowed_param = ['id'], require_auth = True ) @@ -207,7 +206,7 @@ class API(object): create_friendship = bind_api( path = '/friendships/create.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name', 'follow'], require_auth = True ) @@ -216,7 +215,7 @@ class API(object): destroy_friendship = bind_api( path = '/friendships/destroy.json', method = 'DELETE', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -224,14 +223,14 @@ class API(object): """ friendships/exists """ exists_friendship = bind_api( path = '/friendships/exists.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['user_a', 'user_b'] ) """ friendships/show """ show_friendship = bind_api( path = '/friendships/show.json', - parser = parse_friendship, + payload_type = 'friendship', allowed_param = ['source_id', 'source_screen_name', 'target_id', 'target_screen_name'] ) @@ -239,14 +238,14 @@ class API(object): """ friends/ids """ friends_ids = bind_api( path = '/friends/ids.json', - parser = parse_ids, + payload_type = 'ids', allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] ) """ followers/ids """ followers_ids = bind_api( path = '/followers/ids.json', - parser = parse_ids, + payload_type = 'ids', allowed_param = ['id', 'user_id', 'screen_name', 'cursor'] ) @@ -255,7 +254,7 @@ class API(object): try: return bind_api( path = '/account/verify_credentials.json', - parser = parse_user, + payload_type = 'user', require_auth = True )(self) except TweepError: @@ -264,7 +263,7 @@ class API(object): """ account/rate_limit_status """ rate_limit_status = bind_api( path = '/account/rate_limit_status.json', - parser = parse_json + payload_type = 'json' ) """ account/update_delivery_device """ @@ -272,7 +271,7 @@ class API(object): path = '/account/update_delivery_device.json', method = 'POST', allowed_param = ['device'], - parser = parse_user, + payload_type = 'user', require_auth = True ) @@ -280,7 +279,7 @@ class API(object): update_profile_colors = bind_api( path = '/account/update_profile_colors.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['profile_background_color', 'profile_text_color', 'profile_link_color', 'profile_sidebar_fill_color', 'profile_sidebar_border_color'], @@ -293,7 +292,7 @@ class API(object): return bind_api( path = '/account/update_profile_image.json', method = 'POST', - parser = parse_user, + payload_type = 'user', require_auth = True )(self, post_data=post_data, headers=headers) @@ -303,7 +302,7 @@ class API(object): bind_api( path = '/account/update_profile_background_image.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['tile'], require_auth = True )(self, post_data=post_data, headers=headers) @@ -312,7 +311,7 @@ class API(object): update_profile = bind_api( path = '/account/update_profile.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['name', 'url', 'location', 'description'], require_auth = True ) @@ -320,7 +319,7 @@ class API(object): """ favorites """ favorites = bind_api( path = '/favorites.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['id', 'page'] ) @@ -328,7 +327,7 @@ class API(object): create_favorite = bind_api( path = '/favorites/create/{id}.json', method = 'POST', - parser = parse_status, + payload_type = 'status', allowed_param = ['id'], require_auth = True ) @@ -337,7 +336,7 @@ class API(object): destroy_favorite = bind_api( path = '/favorites/destroy/{id}.json', method = 'DELETE', - parser = parse_status, + payload_type = 'status', allowed_param = ['id'], require_auth = True ) @@ -346,7 +345,7 @@ class API(object): enable_notifications = bind_api( path = '/notifications/follow.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -355,7 +354,7 @@ class API(object): disable_notifications = bind_api( path = '/notifications/leave.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -364,7 +363,7 @@ class API(object): create_block = bind_api( path = '/blocks/create.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -373,7 +372,7 @@ class API(object): destroy_block = bind_api( path = '/blocks/destroy.json', method = 'DELETE', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -383,7 +382,6 @@ class API(object): try: bind_api( path = '/blocks/exists.json', - parser = parse_none, allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True )(self, *args, **kargs) @@ -394,7 +392,7 @@ class API(object): """ blocks/blocking """ blocks = bind_api( path = '/blocks/blocking.json', - parser = parse_users, + payload_type = 'user', payload_list = True, allowed_param = ['page'], require_auth = True ) @@ -402,7 +400,7 @@ class API(object): """ blocks/blocking/ids """ blocks_ids = bind_api( path = '/blocks/blocking/ids.json', - parser = parse_json, + payload_type = 'json', require_auth = True ) @@ -410,7 +408,7 @@ class API(object): report_spam = bind_api( path = '/report_spam.json', method = 'POST', - parser = parse_user, + payload_type = 'user', allowed_param = ['id', 'user_id', 'screen_name'], require_auth = True ) @@ -418,14 +416,14 @@ class API(object): """ saved_searches """ saved_searches = bind_api( path = '/saved_searches.json', - parser = parse_saved_searches, + payload_type = 'saved_search', payload_list = True, require_auth = True ) """ saved_searches/show """ get_saved_search = bind_api( path = '/saved_searches/show/{id}.json', - parser = parse_saved_search, + payload_type = 'saved_search', allowed_param = ['id'], require_auth = True ) @@ -434,7 +432,7 @@ class API(object): create_saved_search = bind_api( path = '/saved_searches/create.json', method = 'POST', - parser = parse_saved_search, + payload_type = 'saved_search', allowed_param = ['query'], require_auth = True ) @@ -443,7 +441,7 @@ class API(object): destroy_saved_search = bind_api( path = '/saved_searches/destroy/{id}.json', method = 'DELETE', - parser = parse_saved_search, + payload_type = 'saved_search', allowed_param = ['id'], require_auth = True ) @@ -451,18 +449,18 @@ class API(object): """ help/test """ def test(self): try: - return bind_api( + bind_api( path = '/help/test.json', - parser = parse_return_true )(self) except TweepError: return False + return True def create_list(self, *args, **kargs): return bind_api( path = '/%s/lists.json' % self.auth.get_username(), method = 'POST', - parser = parse_list, + payload_type = 'list', allowed_param = ['name', 'mode', 'description'], require_auth = True )(self, *args, **kargs) @@ -471,7 +469,7 @@ class API(object): return bind_api( path = '/%s/lists/%s.json' % (self.auth.get_username(), slug), method = 'DELETE', - parser = parse_list, + payload_type = 'list', require_auth = True )(self) @@ -479,41 +477,41 @@ class API(object): return bind_api( path = '/%s/lists/%s.json' % (self.auth.get_username(), slug), method = 'POST', - parser = parse_list, + payload_type = 'list', allowed_param = ['name', 'mode', 'description'], require_auth = True )(self, *args, **kargs) lists = bind_api( path = '/{user}/lists.json', - parser = parse_lists, + payload_type = 'list', payload_list = True, allowed_param = ['user', 'cursor'], require_auth = True ) lists_memberships = bind_api( path = '/{user}/lists/memberships.json', - parser = parse_lists, + payload_type = 'list', payload_list = True, allowed_param = ['user', 'cursor'], require_auth = True ) lists_subscriptions = bind_api( path = '/{user}/lists/subscriptions.json', - parser = parse_lists, + payload_type = 'list', payload_list = True, allowed_param = ['user', 'cursor'], require_auth = True ) list_timeline = bind_api( path = '/{owner}/lists/{slug}/statuses.json', - parser = parse_statuses, + payload_type = 'status', payload_list = True, allowed_param = ['owner', 'slug', 'since_id', 'max_id', 'count', 'page'] ) get_list = bind_api( path = '/{owner}/lists/{slug}.json', - parser = parse_list, + payload_type = 'list', allowed_param = ['owner', 'slug'] ) @@ -521,7 +519,7 @@ class API(object): return bind_api( path = '/%s/%s/members.json' % (self.auth.get_username(), slug), method = 'POST', - parser = parse_list, + payload_type = 'list', allowed_param = ['id'], require_auth = True )(self, *args, **kargs) @@ -530,14 +528,14 @@ class API(object): return bind_api( path = '/%s/%s/members.json' % (self.auth.get_username(), slug), method = 'DELETE', - parser = parse_list, + payload_type = 'list', allowed_param = ['id'], require_auth = True )(self, *args, **kargs) list_members = bind_api( path = '/{owner}/{slug}/members.json', - parser = parse_users, + payload_type = 'user', payload_list = True, allowed_param = ['owner', 'slug', 'cursor'] ) @@ -545,7 +543,7 @@ class API(object): try: return bind_api( path = '/%s/%s/members/%s.json' % (owner, slug, user_id), - parser = parse_user + payload_type = 'user' )(self) except TweepError: return False @@ -553,7 +551,7 @@ class API(object): subscribe_list = bind_api( path = '/{owner}/{slug}/subscribers.json', method = 'POST', - parser = parse_list, + payload_type = 'list', allowed_param = ['owner', 'slug'], require_auth = True ) @@ -561,14 +559,14 @@ class API(object): unsubscribe_list = bind_api( path = '/{owner}/{slug}/subscribers.json', method = 'DELETE', - parser = parse_list, + payload_type = 'list', allowed_param = ['owner', 'slug'], require_auth = True ) list_subscribers = bind_api( path = '/{owner}/{slug}/subscribers.json', - parser = parse_users, + payload_type = 'user', payload_list = True, allowed_param = ['owner', 'slug', 'cursor'] ) @@ -576,22 +574,22 @@ class API(object): try: return bind_api( path = '/%s/%s/subscribers/%s.json' % (owner, slug, user_id), - parser = parse_user + payload_type = 'user' )(self) except TweepError: return False - """ trends/available [coming soon] """ + """ trends/available """ trends_available = bind_api( path = '/trends/available.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['lat', 'long'] ) - """ trends/location [coming soon] """ + """ trends/location """ trends_location = bind_api( path = '/trends/{woeid}.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['woeid'] ) @@ -599,7 +597,7 @@ class API(object): search = bind_api( search_api = True, path = '/search.json', - parser = parse_search_results, + payload_type = 'search_result', payload_list = True, allowed_param = ['q', 'lang', 'locale', 'rpp', 'page', 'since_id', 'geocode', 'show_user'] ) search.pagination_mode = 'page' @@ -608,14 +606,14 @@ class API(object): trends = bind_api( search_api = True, path = '/trends.json', - parser = parse_json + payload_type = 'json' ) """ trends/current """ trends_current = bind_api( search_api = True, path = '/trends/current.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['exclude'] ) @@ -623,7 +621,7 @@ class API(object): trends_daily = bind_api( search_api = True, path = '/trends/daily.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['date', 'exclude'] ) @@ -631,7 +629,7 @@ class API(object): trends_weekly = bind_api( search_api = True, path = '/trends/weekly.json', - parser = parse_json, + payload_type = 'json', allowed_param = ['date', 'exclude'] ) diff --git a/tweepy/binder.py b/tweepy/binder.py index 453b024..0a80ca2 100644 --- a/tweepy/binder.py +++ b/tweepy/binder.py @@ -7,25 +7,13 @@ import urllib import time import re -from tweepy.parsers import parse_error from tweepy.error import TweepError -try: - import simplejson as json -except ImportError: - try: - import json # Python 2.6+ - except ImportError: - try: - from django.utils import simplejson as json # Google App Engine - except ImportError: - raise ImportError, "Can't load a json library" - re_path_template = re.compile('{\w+}') -def bind_api(path, parser, allowed_param=[], method='GET', require_auth=False, - timeout=None, search_api = False): +def bind_api(path, payload_type=None, payload_list=False, allowed_param=[], method='GET', + require_auth=False, timeout=None, search_api = False): def _call(api, *args, **kargs): # If require auth, throw exception if credentials not provided @@ -159,36 +147,16 @@ def bind_api(path, parser, allowed_param=[], method='GET', require_auth=False, error_msg = "Twitter error response: status code = %s" % resp.status raise TweepError(error_msg) - # Parse json respone body - try: - jobject = json.loads(resp.read()) - except Exception, e: - raise TweepError("Failed to parse json: %s" % e) - - # Parse cursor infomation - if isinstance(jobject, dict): - next_cursor = jobject.get('next_cursor') - prev_cursor = jobject.get('previous_cursor') - else: - next_cursor = None - prev_cursor = None - - # Pass json object into parser - try: - if parameters and 'cursor' in parameters: - out = parser(jobject, api), next_cursor, prev_cursor - else: - out = parser(jobject, api) - except Exception, e: - raise TweepError("Failed to parse response: %s" % e) + # Parse the response payload + result = api.parser.parse(api, payload_type, payload_list, resp.read()) conn.close() # store result in cache - if api.cache and method == 'GET': - api.cache.store(url, out) + if api.cache and method == 'GET' and result: + api.cache.store(url, result) - return out + return result # Set pagination mode diff --git a/tweepy/models.py b/tweepy/models.py index 776c39f..b64a51c 100644 --- a/tweepy/models.py +++ b/tweepy/models.py @@ -3,10 +3,19 @@ # See LICENSE from tweepy.error import TweepError +from tweepy.utils import parse_datetime, parse_html_value, parse_a_href, \ + parse_search_datetime, unescape_html + + +class ResultSet(list): + """A list like object that holds results from a Twitter API query.""" class Model(object): + def __init__(self, api=None): + self._api = api + def __getstate__(self): # pickle pickle = {} @@ -17,9 +26,44 @@ class Model(object): pickle[k] = v return pickle + @classmethod + def parse(cls, api, json): + """Parse a JSON object into a model instance.""" + raise NotImplementedError + + @classmethod + def parse_list(cls, api, json_list): + """Parse a list of JSON objects into a result set of model instances.""" + results = ResultSet() + for obj in json_list: + results.append(cls.parse(api, obj)) + return results + class Status(Model): + @classmethod + def parse(cls, api, json): + status = cls(api) + for k, v in json.items(): + if k == 'user': + user = User.parse(api, v) + setattr(status, 'author', user) + setattr(status, 'user', user) # DEPRECIATED + elif k == 'created_at': + setattr(status, k, parse_datetime(v)) + elif k == 'source': + if '<' in v: + setattr(status, k, parse_html_value(v)) + setattr(status, 'source_url', parse_a_href(v)) + else: + setattr(status, k, v) + elif k == 'retweeted_status': + setattr(status, k, User.parse(api, v)) + else: + setattr(status, k, v) + return status + def destroy(self): return self._api.destroy_status(self.id) @@ -35,6 +79,36 @@ class Status(Model): class User(Model): + @classmethod + def parse(cls, api, json): + user = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(user, k, parse_datetime(v)) + elif k == 'status': + setattr(user, k, Status.parse(api, v)) + elif k == 'following': + # twitter sets this to null if it is false + if v is True: + setattr(user, k, True) + else: + setattr(user, k, False) + else: + setattr(user, k, v) + return user + + @classmethod + def parse_list(cls, api, json_list): + if isinstance(json_list, list): + item_list = json_list + else: + item_list = json_list['users'] + + results = ResultSet() + for obj in item_list: + results.append(cls.parse(api, obj)) + return results + def timeline(self, **kargs): return self._api.user_timeline(user_id=self.id, **kargs) @@ -67,32 +141,113 @@ class User(Model): class DirectMessage(Model): + @classmethod + def parse(cls, api, json): + dm = cls(api) + for k, v in json.items(): + if k == 'sender' or k == 'recipient': + setattr(dm, k, User.parse(api, v)) + elif k == 'created_at': + setattr(dm, k, parse_datetime(v)) + else: + setattr(dm, k, v) + return dm + def destroy(self): return self._api.destroy_direct_message(self.id) class Friendship(Model): - pass + @classmethod + def parse(cls, api, json): + relationship = json['relationship'] + + # parse source + source = cls(api) + for k, v in relationship['source'].items(): + setattr(source, k, v) + + # parse target + target = cls(api) + for k, v in relationship['target'].items(): + setattr(target, k, v) + + return source, target class SavedSearch(Model): + @classmethod + def parse(cls, api, json): + ss = cls(api) + for k, v in json.items(): + if k == 'created_at': + setattr(ss, k, parse_datetime(v)) + else: + setattr(ss, k, v) + return ss + def destroy(self): return self._api.destroy_saved_search(self.id) class SearchResult(Model): - pass + @classmethod + def parse(cls, api, json): + result = cls() + for k, v in json.items(): + if k == 'created_at': + setattr(result, k, parse_search_datetime(v)) + elif k == 'source': + setattr(result, k, parse_html_value(unescape_html(v))) + else: + setattr(result, k, v) + return result + + @classmethod + def parse_list(cls, api, json_list, result_set=None): + results = ResultSet() + results.max_id = json_list.get('max_id') + results.since_id = json_list.get('since_id') + results.refresh_url = json_list.get('refresh_url') + results.next_page = json_list.get('next_page') + results.results_per_page = json_list.get('results_per_page') + results.page = json_list.get('page') + results.completed_in = json_list.get('completed_in') + results.query = json_list.get('query') + + for obj in json_list['results']: + results.append(cls.parse(api, obj)) + return results + class Retweet(Model): + #TODO: remove me def destroy(self): return self._api.destroy_status(self.id) class List(Model): + @classmethod + def parse(cls, api, json): + lst = List(api) + for k,v in json.items(): + if k == 'user': + setattr(lst, k, User.parse(api, v)) + else: + setattr(lst, k, v) + return lst + + @classmethod + def parse_list(cls, api, json_list, result_set=None): + results = ResultSet() + for obj in json_list['lists']: + results.append(cls.parse(api, obj)) + return results + def update(self, **kargs): return self._api.update_list(self.slug, **kargs) @@ -127,6 +282,23 @@ class List(Model): return self._api.is_subscribed_list(self.user.screen_name, self.slug, id) +class JSONModel(Model): + + @classmethod + def parse(cls, api, json): + return json + + +class IDModel(Model): + + @classmethod + def parse(cls, api, json): + if isinstance(json, list): + return json + else: + return json['ids'] + + class ModelFactory(object): """ Used by parsers for creating instances @@ -143,3 +315,6 @@ class ModelFactory(object): retweet = Retweet list = List + json = JSONModel + ids = IDModel + diff --git a/tweepy/parsers.py b/tweepy/parsers.py index 16ee1dc..2f7177e 100644 --- a/tweepy/parsers.py +++ b/tweepy/parsers.py @@ -2,265 +2,39 @@ # Copyright 2009 Joshua Roesslein # See LICENSE -import htmlentitydefs -import re -from datetime import datetime -import time +from tweepy.models import ModelFactory +from tweepy.utils import import_simplejson -class ResultSet(list): - """A list like object that holds results from a Twitter API query.""" +class Parser(object): + payload_format = 'json' -def _parse_cursor(obj): + def parse(self, api, payload_type, payload_list, payload): + """Parse the response payload and return the result.""" + raise NotImplementedError - return obj['next_cursor'], obj['prev_cursor'] -def parse_json(obj, api): +class ModelParser(Parser): - return obj + def __init__(self, model_factory=None): + self.model_factory = model_factory or ModelFactory + self.json_lib = import_simplejson() + def parse(self, api, payload_type, payload_list, payload): + try: + if payload_type is None: return + model = getattr(self.model_factory, payload_type) + except AttributeError: + raise TweepError('No model for this payload type: %s' % method.payload_type) -def parse_return_true(obj, api): + try: + json = self.json_lib.loads(payload) + except Exception, e: + raise TweepError('Failed to parse JSON: %s' % e) - return True - - -def parse_none(obj, api): - - return None - - -def parse_error(obj): - - return obj['error'] - - -def _parse_datetime(str): - - # We must parse datetime this way to work in python 2.4 - return datetime(*(time.strptime(str, '%a %b %d %H:%M:%S +0000 %Y')[0:6])) - - -def _parse_search_datetime(str): - - # python 2.4 - return datetime(*(time.strptime(str, '%a, %d %b %Y %H:%M:%S +0000')[0:6])) - - -def unescape_html(text): - """Created by Fredrik Lundh (http://effbot.org/zone/re-sub.htm#unescape-html)""" - def fixup(m): - text = m.group(0) - if text[:2] == "&#": - # character reference - try: - if text[:3] == "&#x": - return unichr(int(text[3:-1], 16)) - else: - return unichr(int(text[2:-1])) - except ValueError: - pass - else: - # named entity - try: - text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) - except KeyError: - pass - return text # leave as is - return re.sub("&#?\w+;", fixup, text) - - -def _parse_html_value(html): - - return html[html.find('>')+1:html.rfind('<')] - - -def _parse_a_href(atag): - - start = atag.find('"') + 1 - end = atag.find('"', start) - return atag[start:end] - - -def parse_user(obj, api): - - user = api.model_factory.user() - user._api = api - for k, v in obj.items(): - if k == 'created_at': - setattr(user, k, _parse_datetime(v)) - elif k == 'status': - setattr(user, k, parse_status(v, api)) - elif k == 'following': - # twitter sets this to null if it is false - if v is True: - setattr(user, k, True) - else: - setattr(user, k, False) - else: - setattr(user, k, v) - return user - - -def parse_users(obj, api): - - if isinstance(obj, list) is False: - item_list = obj['users'] - else: - item_list = obj - - users = ResultSet() - for item in item_list: - if item is None: break # sometimes an empty list with a null in it - users.append(parse_user(item, api)) - return users - - -def parse_status(obj, api): - - status = api.model_factory.status() - status._api = api - for k, v in obj.items(): - if k == 'user': - user = parse_user(v, api) - setattr(status, 'author', user) - setattr(status, 'user', user) # DEPRECIATED - elif k == 'created_at': - setattr(status, k, _parse_datetime(v)) - elif k == 'source': - if '<' in v: - setattr(status, k, _parse_html_value(v)) - setattr(status, 'source_url', _parse_a_href(v)) - else: - setattr(status, k, v) - elif k == 'retweeted_status': - setattr(status, k, parse_status(v, api)) + if payload_list: + return model.parse_list(api, json) else: - setattr(status, k, v) - return status - - -def parse_statuses(obj, api): - - statuses = ResultSet() - for item in obj: - statuses.append(parse_status(item, api)) - return statuses - - -def parse_dm(obj, api): - - dm = api.model_factory.direct_message() - dm._api = api - for k, v in obj.items(): - if k == 'sender' or k == 'recipient': - setattr(dm, k, parse_user(v, api)) - elif k == 'created_at': - setattr(dm, k, _parse_datetime(v)) - else: - setattr(dm, k, v) - return dm - - -def parse_directmessages(obj, api): - - directmessages = ResultSet() - for item in obj: - directmessages.append(parse_dm(item, api)) - return directmessages - - -def parse_friendship(obj, api): - - relationship = obj['relationship'] - - # parse source - source = api.model_factory.friendship() - for k, v in relationship['source'].items(): - setattr(source, k, v) - - # parse target - target = api.model_factory.friendship() - for k, v in relationship['target'].items(): - setattr(target, k, v) - - return source, target - - -def parse_ids(obj, api): - - if isinstance(obj, list) is False: - return obj['ids'] - else: - return obj - -def parse_saved_search(obj, api): - - ss = api.model_factory.saved_search() - ss._api = api - for k, v in obj.items(): - if k == 'created_at': - setattr(ss, k, _parse_datetime(v)) - else: - setattr(ss, k, v) - return ss - - -def parse_saved_searches(obj, api): - - saved_searches = ResultSet() - saved_search = api.model_factory.saved_search() - for item in obj: - saved_searches.append(parse_saved_search(item, api)) - return saved_searches - - -def parse_search_result(obj, api): - - result = api.model_factory.search_result() - for k, v in obj.items(): - if k == 'created_at': - setattr(result, k, _parse_search_datetime(v)) - elif k == 'source': - setattr(result, k, _parse_html_value(unescape_html(v))) - else: - setattr(result, k, v) - return result - - -def parse_search_results(obj, api): - - results = ResultSet() - results.max_id = obj.get('max_id') - results.since_id = obj.get('since_id') - results.refresh_url = obj.get('refresh_url') - results.next_page = obj.get('next_page') - results.results_per_page = obj.get('results_per_page') - results.page = obj.get('page') - results.completed_in = obj.get('completed_in') - results.query = obj.get('query') - - for item in obj['results']: - results.append(parse_search_result(item, api)) - return results - - -def parse_list(obj, api): - - lst = api.model_factory.list() - lst._api = api - for k,v in obj.items(): - if k == 'user': - setattr(lst, k, parse_user(v, api)) - else: - setattr(lst, k, v) - return lst - -def parse_lists(obj, api): - - lists = ResultSet() - for item in obj['lists']: - lists.append(parse_list(item, api)) - return lists + return model.parse(api, json) diff --git a/tweepy/streaming.py b/tweepy/streaming.py index bd871c6..e576ec5 100644 --- a/tweepy/streaming.py +++ b/tweepy/streaming.py @@ -9,7 +9,7 @@ from time import sleep import urllib from tweepy.auth import BasicAuthHandler -from tweepy.parsers import parse_status +from tweepy.models import Status from tweepy.api import API from tweepy.error import TweepError @@ -40,7 +40,7 @@ class StreamListener(object): """ if 'in_reply_to_status_id' in data: - status = parse_status(json.loads(data), self.api) + status = Status.parse(self.api, json.loads(data)) if self.on_status(status) is False: return False elif 'delete' in data: diff --git a/tweepy/utils.py b/tweepy/utils.py new file mode 100644 index 0000000..3608f6f --- /dev/null +++ b/tweepy/utils.py @@ -0,0 +1,71 @@ +# Tweepy +# Copyright 2010 Joshua Roesslein +# See LICENSE + +from datetime import datetime +import time +import htmlentitydefs +import re + + +def parse_datetime(str): + + # We must parse datetime this way to work in python 2.4 + return datetime(*(time.strptime(str, '%a %b %d %H:%M:%S +0000 %Y')[0:6])) + + +def parse_html_value(html): + + return html[html.find('>')+1:html.rfind('<')] + + +def parse_a_href(atag): + + start = atag.find('"') + 1 + end = atag.find('"', start) + return atag[start:end] + + +def parse_search_datetime(str): + + # python 2.4 + return datetime(*(time.strptime(str, '%a, %d %b %Y %H:%M:%S +0000')[0:6])) + + +def unescape_html(text): + """Created by Fredrik Lundh (http://effbot.org/zone/re-sub.htm#unescape-html)""" + def fixup(m): + text = m.group(0) + if text[:2] == "&#": + # character reference + try: + if text[:3] == "&#x": + return unichr(int(text[3:-1], 16)) + else: + return unichr(int(text[2:-1])) + except ValueError: + pass + else: + # named entity + try: + text = unichr(htmlentitydefs.name2codepoint[text[1:-1]]) + except KeyError: + pass + return text # leave as is + return re.sub("&#?\w+;", fixup, text) + + +def import_simplejson(): + try: + import simplejson as json + except ImportError: + try: + import json # Python 2.6+ + except ImportError: + try: + from django.utils import simplejson as json # Google App Engine + except ImportError: + raise ImportError, "Can't load a json library" + + return json + -- 2.25.1