From cb3f22af66ddf9ddca3b57ff0d6d24a51902ff42 Mon Sep 17 00:00:00 2001 From: Harmon Date: Thu, 5 Dec 2019 20:12:19 -0600 Subject: [PATCH] Add cursor support for API.search_30_day and API.search_full_archive Adds pagination decorator and NextIterator --- tweepy/api.py | 4 +++- tweepy/binder.py | 10 +++++++++- tweepy/cursor.py | 24 ++++++++++++++++++++++++ tweepy/parsers.py | 4 +++- 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/tweepy/api.py b/tweepy/api.py index eb442e1..1836ee3 100644 --- a/tweepy/api.py +++ b/tweepy/api.py @@ -7,7 +7,7 @@ import os import six -from tweepy.binder import bind_api +from tweepy.binder import bind_api, pagination from tweepy.error import TweepError from tweepy.parsers import ModelParser, Parser from tweepy.utils import list_to_csv @@ -1280,6 +1280,7 @@ class API(object): 'include_entities'] ) + @pagination(mode='next') def search_30_day(self, environment_name, *args, **kwargs): """ :reference: https://developer.twitter.com/en/docs/tweets/search/api-reference/premium-search :allowed_param: 'query', 'tag', 'fromDate', 'toDate', 'maxResults', @@ -1294,6 +1295,7 @@ class API(object): require_auth=True )(*args, **kwargs) + @pagination(mode='next') def search_full_archive(self, environment_name, *args, **kwargs): """ :reference: https://developer.twitter.com/en/docs/tweets/search/api-reference/premium-search :allowed_param: 'query', 'tag', 'fromDate', 'toDate', 'maxResults', diff --git a/tweepy/binder.py b/tweepy/binder.py index 846cfbf..88f98a4 100644 --- a/tweepy/binder.py +++ b/tweepy/binder.py @@ -234,7 +234,8 @@ def bind_api(**config): raise TweepError(error_msg, resp, api_code=api_error_code) # Parse the response payload - self.return_cursors = self.return_cursors or 'cursor' in self.session.params + self.return_cursors = (self.return_cursors or + 'cursor' in self.session.params or 'next' in self.session.params) result = self.parser.parse(self, resp.text, return_cursors=self.return_cursors) # Store result into cache if one is available. @@ -266,3 +267,10 @@ def bind_api(**config): _call.pagination_mode = 'page' return _call + + +def pagination(mode): + def decorator(method): + method.pagination_mode = mode + return method + return decorator diff --git a/tweepy/cursor.py b/tweepy/cursor.py index 2a3d950..fe76fc5 100644 --- a/tweepy/cursor.py +++ b/tweepy/cursor.py @@ -17,6 +17,8 @@ class Cursor(object): self.iterator = DMCursorIterator(method, *args, **kwargs) elif method.pagination_mode == 'id': self.iterator = IdIterator(method, *args, **kwargs) + elif method.pagination_mode == "next": + self.iterator = NextIterator(method, *args, **kwargs) elif method.pagination_mode == 'page': self.iterator = PageIterator(method, *args, **kwargs) else: @@ -201,6 +203,28 @@ class PageIterator(BaseIterator): return self.method(page=self.current_page, *self.args, **self.kwargs) +class NextIterator(BaseIterator): + + def __init__(self, method, *args, **kwargs): + BaseIterator.__init__(self, method, *args, **kwargs) + self.next_token = self.kwargs.pop('next', None) + self.page_count = 0 + + def next(self): + if self.next_token == -1 or (self.limit and self.page_count == self.limit): + raise StopIteration + data = self.method(next=self.next_token, return_cursors=True, *self.args, **self.kwargs) + self.page_count += 1 + if isinstance(data, tuple): + data, self.next_token = data + else: + self.next_token = -1 + return data + + def prev(self): + raise TweepError('This method does not allow backwards pagination') + + class ItemIterator(BaseIterator): def __init__(self, page_iterator): diff --git a/tweepy/parsers.py b/tweepy/parsers.py index 7d09f63..a047fc4 100644 --- a/tweepy/parsers.py +++ b/tweepy/parsers.py @@ -50,7 +50,9 @@ class JSONParser(Parser): raise TweepError('Failed to parse JSON payload: %s' % e) if return_cursors and isinstance(json, dict): - if 'next_cursor' in json: + if 'next' in json: + return json, json['next'] + elif 'next_cursor' in json: if 'previous_cursor' in json: cursors = json['previous_cursor'], json['next_cursor'] return json, cursors -- 2.25.1