From a875f270a83dd9003b577f6d9922cf32835e1514 Mon Sep 17 00:00:00 2001 From: Ethan Baer Date: Fri, 31 Jul 2015 15:03:26 -0500 Subject: [PATCH 1/3] fixed cursor function code to return next pages when funciton iter_mode is 'id' --- twython/api.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/twython/api.py b/twython/api.py index d2d5c03f..6046f131 100644 --- a/twython/api.py +++ b/twython/api.py @@ -18,6 +18,7 @@ from . import __version__ from .advisory import TwythonDeprecationWarning from .compat import json, urlencode, parse_qsl, quote_plus, str, is_py2 +from .compat import urlsplit from .endpoints import EndpointsMixin from .exceptions import TwythonError, TwythonAuthError, TwythonRateLimitError from .helpers import _transparent_params @@ -498,19 +499,27 @@ def cursor(self, function, return_pages=False, **params): try: if function.iter_mode == 'id': - if 'max_id' not in params: - # Add 1 to the id because since_id and - # max_id are inclusive - if hasattr(function, 'iter_metadata'): - since_id = content[function.iter_metadata].get('since_id_str') + # Set max_id in params to one less than lowest tweet id + if hasattr(function, 'iter_metadata'): + # Get supplied next max_id + metadata = content.get(function.iter_metadata) + if 'next_results' in metadata: + next_results = urlsplit(metadata['next_results']) + params = dict(parse_qsl(next_results.query)) else: - since_id = content[0]['id_str'] - params['since_id'] = (int(since_id) - 1) + # No more results + raise StopIteration + else: + # Twitter gives tweets in reverse chronological order: + params['max_id'] = str(int(content[-1]['id_str']) - 1) elif function.iter_mode == 'cursor': params['cursor'] = content['next_cursor_str'] except (TypeError, ValueError): # pragma: no cover raise TwythonError('Unable to generate next page of search \ results, `page` is not a number.') + except (KeyError, AttributeError): #pragma no cover + raise TwythonError('Unable to generate next page of search \ + results, content has unexpected structure.') @staticmethod def unicode2utf8(text): From 0b3df413d825af9026182a3fb96febde0a050a95 Mon Sep 17 00:00:00 2001 From: Ethan Baer Date: Fri, 31 Jul 2015 15:07:25 -0500 Subject: [PATCH 2/3] added urlsplit import for use in api.cursor function --- twython/compat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/twython/compat.py b/twython/compat.py index c36b3269..7c049b00 100644 --- a/twython/compat.py +++ b/twython/compat.py @@ -25,7 +25,7 @@ if is_py2: from urllib import urlencode, quote_plus - from urlparse import parse_qsl + from urlparse import parse_qsl, urlsplit str = unicode basestring = basestring @@ -33,7 +33,7 @@ elif is_py3: - from urllib.parse import urlencode, quote_plus, parse_qsl + from urllib.parse import urlencode, quote_plus, parse_qsl, urlsplit str = str basestring = (str, bytes) From 9e51c7fba9179fb6db5fe131df3ed574a94f08d1 Mon Sep 17 00:00:00 2001 From: Ethan Baer Date: Thu, 6 Aug 2015 17:11:00 -0500 Subject: [PATCH 3/3] added session rebuilding to test possible fix for requests.ConnectionError happening after long sleep --- twython/api.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/twython/api.py b/twython/api.py index 6046f131..321cf634 100644 --- a/twython/api.py +++ b/twython/api.py @@ -8,8 +8,8 @@ Twitter Authentication, and miscellaneous methods that are useful when dealing with the Twitter API """ - import warnings +import logging import requests from requests.auth import HTTPBasicAuth @@ -25,7 +25,6 @@ warnings.simplefilter('always', TwythonDeprecationWarning) # For Python 2.7 > - class Twython(EndpointsMixin, object): def __init__(self, app_key=None, app_secret=None, oauth_token=None, oauth_token_secret=None, access_token=None, @@ -163,7 +162,23 @@ def _request(self, url, method='GET', params=None, api_call=None): try: response = func(url, **requests_args) except requests.RequestException as e: - raise TwythonError(str(e)) + logging.exception('RequestException encountered - refreshing...') + # Rebuild Session and retry + s2 = requests.Session() + s2.auth = self.client.auth + s2.headers = self.client.headers + s2.cert = self.client.cert + s2.hooks = self.client.hooks + s2.proxies = self.client.proxies + s2.max_redirects = self.client.max_redirects + self.client.close() + + self.client = s2 + try: + response = func(url, **requests_args) + except requests.RequestException as e: + logging.exception('RequestException encountered after refresh!') + raise TwythonError(str(e)) # create stash for last function intel self._last_call = {