From 9e06f3e96d4b48d63f76280585d72d6a3772f089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Mah=C3=A9?= Date: Thu, 6 Apr 2017 14:34:52 -0700 Subject: [PATCH] Implementing basic link pagination fetch issue #4 --- mastodon/Mastodon.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mastodon/Mastodon.py b/mastodon/Mastodon.py index 9967bdb..117b81f 100644 --- a/mastodon/Mastodon.py +++ b/mastodon/Mastodon.py @@ -303,7 +303,7 @@ class Mastodon: Returns a list of user dicts. """ - return self.__api_request('GET', '/api/v1/accounts/' + str(id) + '/following') + return self.__api_request('GET', '/api/v1/accounts/' + str(id) + '/following',do_fetch_all=True) def account_followers(self, id): """ @@ -598,12 +598,13 @@ class Mastodon: return (date_time_utc - epoch_utc).total_seconds() - def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True): + def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True, do_fetch_all = False): """ Internal API request helper. """ response = None headers = None + next_url = None # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it # would take to not hit the rate limit at that request rate. @@ -661,7 +662,7 @@ class Mastodon: if self.debug_requests == True: print('Mastodon: Response received with code ' + str(response_object.status_code) + '.') print('response headers: ' + str(response_object.headers)) - print('Response text content: ' + str(response_object.text)) + #print('Response text content: ' + str(response_object.text)) if response_object.status_code == 404: raise MastodonAPIError('Endpoint not found.') @@ -676,6 +677,18 @@ class Mastodon: traceback.print_exc() raise MastodonAPIError("Could not parse response as JSON, response code was %s, bad json content was '%s'" % (response_object.status_code, response_object.content)) + if 'Link' in response_object.headers and do_fetch_all: + tmp_url = requests.utils.parse_header_links(response_object.headers['Link'].rstrip('>').replace('>,<', ',<')) + if tmp_url: + for url in tmp_url: + if url['rel'] == 'next': + next_url = url['url'].replace(self.api_base_url,'') + break + if next_url is not None: + tmp_response = self.__api_request(method, next_url, params = params, files = files, do_ratelimiting = do_ratelimiting, do_fetch_all = True) + if type(tmp_response) == type(response): + response += tmp_response + # Handle rate limiting if 'X-RateLimit-Remaining' in response_object.headers and do_ratelimiting: self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining'])