From e37afb6324c67899683323313043264383b9e1a0 Mon Sep 17 00:00:00 2001 From: Ansem Date: Mon, 10 Apr 2017 22:48:30 +0000 Subject: [PATCH] Initial commit for pagination support --- mastodon/Mastodon.py | 52 ++++-------- mastodon/__init__.py | 10 ++- mastodon/exceptions.py | 19 +++++ mastodon/response.py | 100 ++++++++++++++++++++++ mastodon/utils.py | 184 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 325 insertions(+), 40 deletions(-) create mode 100644 mastodon/exceptions.py create mode 100644 mastodon/response.py create mode 100644 mastodon/utils.py diff --git a/mastodon/Mastodon.py b/mastodon/Mastodon.py index eba1f0e..dfbdb2c 100644 --- a/mastodon/Mastodon.py +++ b/mastodon/Mastodon.py @@ -17,6 +17,9 @@ import requests import dateutil import dateutil.parser +from mastodon.exceptions import * +from mastodon.response import ResponseObject + class Mastodon: """ Super basic but thorough and easy to use mastodon.social @@ -718,13 +721,12 @@ class Mastodon: return (date_time_utc - epoch_utc).total_seconds() - def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True, do_fetch_all = False): + def __api_request(self, method, endpoint, params={}, files={}, do_ratelimiting=True, get_r_object=False): """ Internal API request helper. """ response = None headers = None - next_url = None # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it # would take to not hit the rate limit at that request rate. @@ -746,10 +748,10 @@ class Mastodon: time.sleep(to_next) # Generate request headers - if self.access_token != None: + if self.access_token is not None: headers = {'Authorization': 'Bearer ' + self.access_token} - if self.debug_requests == True: + if self.debug_requests is True: print('Mastodon: Request to endpoint "' + endpoint + '" using method "' + method + '".') print('Parameters: ' + str(params)) print('Headers: ' + str(headers)) @@ -791,24 +793,19 @@ class Mastodon: raise MastodonAPIError('General API problem.') try: - response = response_object.json() + if get_r_object: + response = ResponseObject._load(response_object, method, params, files, do_ratelimiting, self.api_base_url) + else: + temp_r = ResponseObject(response_object.json(), response_object, method, params, files, do_ratelimiting, self.api_base_url) + if temp_r is not None: + response = temp_r.response + else: + print("Big error") except: import traceback traceback.print_exc() raise MastodonAPIError("Could not parse response as JSON, response code was %s, bad json content was '%s'" % (response_object.status_code, response_object.content)) - if 'Link' in response_object.headers and do_fetch_all: - tmp_url = requests.utils.parse_header_links(response_object.headers['Link'].rstrip('>').replace('>,<', ',<')) - if tmp_url: - for url in tmp_url: - if url['rel'] == 'next': - next_url = url['url'].replace(self.api_base_url,'') - break - if next_url is not None: - tmp_response = self.__api_request(method, next_url, params = params, files = files, do_ratelimiting = do_ratelimiting, do_fetch_all = True) - if type(tmp_response) == type(response): - response += tmp_response - # Handle rate limiting if 'X-RateLimit-Remaining' in response_object.headers and do_ratelimiting: self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining']) @@ -837,9 +834,9 @@ class Mastodon: to_next = self.ratelimit_reset - time.time() if to_next > 0: # As a precaution, never sleep longer than 5 minutes + request_complete = False to_next = min(to_next, 5 * 60) time.sleep(to_next) - request_complete = False return response @@ -881,22 +878,3 @@ class Mastodon: del params[key] return params - -## -# Exceptions -## -class MastodonIllegalArgumentError(ValueError): - pass - -class MastodonFileNotFoundError(IOError): - pass - -class MastodonNetworkError(IOError): - pass - -class MastodonAPIError(Exception): - pass - -class MastodonRatelimitError(Exception): - pass - diff --git a/mastodon/__init__.py b/mastodon/__init__.py index 9c8e39b..65c6104 100644 --- a/mastodon/__init__.py +++ b/mastodon/__init__.py @@ -1,4 +1,8 @@ -from mastodon.Mastodon import Mastodon -from mastodon.streaming import StreamListener, MalformedEventError +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- -__all__ = ['Mastodon', 'StreamListener', 'MalformedEventError'] +from mastodon.Mastodon import Mastodon +from mastodon.exceptions import * +from mastodon.response import ResponseObject +from mastodon.utils import * +from mastodon.streaming import StreamListener, MalformedEventError diff --git a/mastodon/exceptions.py b/mastodon/exceptions.py new file mode 100644 index 0000000..fa0cc58 --- /dev/null +++ b/mastodon/exceptions.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Exceptions""" + +class MastodonIllegalArgumentError(ValueError): + pass + +class MastodonFileNotFoundError(IOError): + pass + +class MastodonNetworkError(IOError): + pass + +class MastodonAPIError(Exception): + pass + +class MastodonRatelimitError(Exception): + pass diff --git a/mastodon/response.py b/mastodon/response.py new file mode 100644 index 0000000..958e5ef --- /dev/null +++ b/mastodon/response.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import json + +import requests + +from mastodon.exceptions import * +import Mastodon + + +class ResponseObject(object): + def __init__(self, response: dict, response_object: requests.models.Response, method: str, params: dict, files: dict, do_ratelimiting: bool, api_base_url: str): + self._response = None + self.response = response + self.response_object = response_object + self.method = method + self.params = params + self.files = files + self.do_ratelimiting = do_ratelimiting + self.api_base_url = api_base_url + self.temp_m = Mastodon("temp") + + + @classmethod + def _load(cls, response: requests.models.Response, method: str = "GET", params: dict = {}, files = {}, do_ratelimiting = True, api_base_url: str = 'https://mastodon.social'): + if type(response) is requests.models.Response: + try: + r = response.json() + print("worked") + return cls(r, response, method, params, files, do_ratelimiting, api_base_url) + except Exception: + return None + else: + return None + + + def __iter__(self): + try: + while True: + if 'Link' in self.response_object.headers: + tmp_url = requests.utils.parse_header_links(self.response_object.headers['Link'].rstrip('>').replace('>,<', ',<')) + next_url = None + if tmp_url: + for url in tmp_url: + if url['rel'] == 'next': + next_url = url['url'].replace(self.api_base_url,'') + break + if next_url is not None: + tmp_response = self.temp_m.__api_request(self.method, next_url, params=self.params, files=self.files, do_ratelimiting=self.do_ratelimiting) + if type(tmp_response) is dict: + self.response = tmp_response + yield tmp_response + else: + return + else: + self.response = self.response_object.json() + return self.response_object.json() + except Exception as e: + raise e + + + def __repr__(self): + return '' % (self.response_object.url) + + def __str__(self): + if self.response is None: + return "" + elif type(self.response) is str: + return self.response + elif type(self.response) is list: + return "\n".join(self.response) + elif type(self.response) is dict: + return json.dumps(self.response) + else: + return str(self.response) + + def __dict__(self): + if type(self.response) is not dict: + return {"response": None} + else: + return {"response": self.response} + + @property + def response(self) -> dict: + return self._response + + @response.setter + def response(self, value: dict): + self._response = value + return + + def fetch_all(self): + r = [] + for page in self: + if page is not None: + r.append(page) + else: + return self.response + return r \ No newline at end of file diff --git a/mastodon/utils.py b/mastodon/utils.py new file mode 100644 index 0000000..3f55eef --- /dev/null +++ b/mastodon/utils.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import time +import datetime +from contextlib import closing + +import pytz +import dateutil +import requests + +from mastodon.exceptions import * + + +"""Internal helpers, dragons probably""" + +def __datetime_to_epoch(self, date_time): + """ + Converts a python datetime to unix epoch, accounting for + time zones and such. + + Assumes UTC if timezone is not given. + """ + date_time_utc = None + if date_time.tzinfo == None: + date_time_utc = date_time.replace(tzinfo = pytz.utc) + else: + date_time_utc = date_time.astimezone(pytz.utc) + + epoch_utc = datetime.datetime.utcfromtimestamp(0).replace(tzinfo = pytz.utc) + + return (date_time_utc - epoch_utc).total_seconds() + +def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True): + """ + Internal API request helper. + """ + response = None + headers = None + next_url = None + + # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it + # would take to not hit the rate limit at that request rate. + if do_ratelimiting and self.ratelimit_method == "pace": + if self.ratelimit_remaining == 0: + to_next = self.ratelimit_reset - time.time() + if to_next > 0: + # As a precaution, never sleep longer than 5 minutes + to_next = min(to_next, 5 * 60) + time.sleep(to_next) + else: + time_waited = time.time() - self.ratelimit_lastcall + time_wait = float(self.ratelimit_reset - time.time()) / float(self.ratelimit_remaining) + remaining_wait = time_wait - time_waited + + if remaining_wait > 0: + to_next = remaining_wait / self.ratelimit_pacefactor + to_next = min(to_next, 5 * 60) + time.sleep(to_next) + + # Generate request headers + if self.access_token is not None: + headers = {'Authorization': 'Bearer ' + self.access_token} + + if self.debug_requests is True: + print('Mastodon: Request to endpoint "' + endpoint + '" using method "' + method + '".') + print('Parameters: ' + str(params)) + print('Headers: ' + str(headers)) + print('Files: ' + str(files)) + + # Make request + request_complete = False + while not request_complete: + request_complete = True + + response_object = None + try: + if method == 'GET': + response_object = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout) + + if method == 'POST': + response_object = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout) + + if method == 'DELETE': + response_object = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout) + except Exception as e: + import traceback + traceback.print_exc() + raise MastodonNetworkError("Could not complete request: %s" % e) + + if response_object == None: + raise MastodonIllegalArgumentError("Illegal request.") + + # Handle response + if self.debug_requests == True: + print('Mastodon: Response received with code ' + str(response_object.status_code) + '.') + print('response headers: ' + str(response_object.headers)) + #print('Response text content: ' + str(response_object.text)) + + if response_object.status_code == 404: + raise MastodonAPIError('Endpoint not found.') + + if response_object.status_code == 500: + raise MastodonAPIError('General API problem.') + + try: + response = response_object.json() + except: + import traceback + traceback.print_exc() + raise MastodonAPIError("Could not parse response as JSON, response code was %s, bad json content was '%s'" % (response_object.status_code, response_object.content)) + + # Handle rate limiting + if 'X-RateLimit-Remaining' in response_object.headers and do_ratelimiting: + self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining']) + self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit']) + + try: + ratelimit_reset_datetime = dateutil.parser.parse(response_object.headers['X-RateLimit-Reset']) + self.ratelimit_reset = self.__datetime_to_epoch(ratelimit_reset_datetime) + + # Adjust server time to local clock + server_time_datetime = dateutil.parser.parse(response_object.headers['Date']) + server_time = self.__datetime_to_epoch(server_time_datetime) + server_time_diff = time.time() - server_time + self.ratelimit_reset += server_time_diff + self.ratelimit_lastcall = time.time() + except Exception as e: + import traceback + traceback.print_exc() + raise MastodonRatelimitError("Rate limit time calculations failed: %s" % e) + + if "error" in response and response["error"] == "Throttled": + if self.ratelimit_method == "throw": + raise MastodonRatelimitError("Hit rate limit.") + + if self.ratelimit_method == "wait" or self.ratelimit_method == "pace": + to_next = self.ratelimit_reset - time.time() + if to_next > 0: + # As a precaution, never sleep longer than 5 minutes + request_complete = False + to_next = min(to_next, 5 * 60) + time.sleep(to_next) + + return response + +def __stream(self, endpoint, listener, params = {}): + """ + Internal streaming API helper. + """ + + headers = {} + if self.access_token != None: + headers = {'Authorization': 'Bearer ' + self.access_token} + + url = self.api_base_url + endpoint + with closing(requests.get(url, headers = headers, data = params, stream = True)) as r: + listener.handle_stream(r.iter_lines()) + + +def __generate_params(self, params, exclude = []): + """ + Internal named-parameters-to-dict helper. + + Note for developers: If called with locals() as params, + as is the usual practice in this code, the __generate_params call + (or at least the locals() call) should generally be the first thing + in your function. + """ + params = dict(params) + + del params['self'] + param_keys = list(params.keys()) + for key in param_keys: + if params[key] == None or key in exclude: + del params[key] + + param_keys = list(params.keys()) + for key in param_keys: + if isinstance(params[key], list): + params[key + "[]"] = params[key] + del params[key] + + return params