Merge 67b64e86ba
into ccaaea0072
This commit is contained in:
commit
f1b31926fe
|
@ -2,18 +2,23 @@
|
|||
|
||||
|
||||
import os
|
||||
from urllib.parse import urlencode
|
||||
import os.path
|
||||
import mimetypes
|
||||
import time
|
||||
import random
|
||||
import string
|
||||
import pytz
|
||||
import datetime
|
||||
from contextlib import closing
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
import dateutil
|
||||
import dateutil.parser
|
||||
from contextlib import closing
|
||||
import requests
|
||||
|
||||
from mastodon.exceptions import *
|
||||
from mastodon.response import ResponseObject
|
||||
|
||||
|
||||
class Mastodon:
|
||||
"""
|
||||
|
@ -48,6 +53,7 @@ class Mastodon:
|
|||
|
||||
Returns client_id and client_secret.
|
||||
"""
|
||||
|
||||
request_data = {
|
||||
'client_name': client_name,
|
||||
'scopes': " ".join(scopes)
|
||||
|
@ -154,9 +160,9 @@ class Mastodon:
|
|||
self._refresh_token = value
|
||||
return
|
||||
|
||||
def auth_request_url(self, client_id: str = None, redirect_uris: str = "urn:ietf:wg:oauth:2.0:oob") -> str:
|
||||
def auth_request_url(self, client_id: str = None, redirect_uris: str = "urn:ietf:wg:oauth:2.0:oob", scopes: list = ['read', 'write', 'follow']) -> str:
|
||||
"""Returns the url that a client needs to request the grant from the server.
|
||||
https://mastodon.social/oauth/authorize?client_id=XXX&response_type=code&redirect_uris=YYY
|
||||
https://mastodon.social/oauth/authorize?client_id=XXX&response_type=code&redirect_uris=YYY&scope=read+write+follow
|
||||
"""
|
||||
if client_id is None:
|
||||
client_id = self.client_id
|
||||
|
@ -169,6 +175,7 @@ class Mastodon:
|
|||
params['client_id'] = client_id
|
||||
params['response_type'] = "code"
|
||||
params['redirect_uri'] = redirect_uris
|
||||
params['scope'] = " ".join(scopes)
|
||||
formatted_params = urlencode(params)
|
||||
return "".join([self.api_base_url, "/oauth/authorize?", formatted_params])
|
||||
|
||||
|
@ -203,7 +210,7 @@ class Mastodon:
|
|||
params = self.__generate_params(locals(), ['scopes', 'to_file', 'username', 'password', 'code'])
|
||||
params['grant_type'] = 'refresh_token'
|
||||
else:
|
||||
raise MastodonIllegalArgumentError('Invalid user name, password, redirect_uris or scopes')
|
||||
raise MastodonIllegalArgumentError('Invalid arguments given. username and password or code are required.')
|
||||
|
||||
params['client_id'] = self.client_id
|
||||
params['client_secret'] = self.client_secret
|
||||
|
@ -216,7 +223,12 @@ class Mastodon:
|
|||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise MastodonIllegalArgumentError('Invalid user name, password, redirect_uris or scopes: %s' % e)
|
||||
if username is not None or password is not None:
|
||||
raise MastodonIllegalArgumentError('Invalid user name, password, or redirect_uris: %s' % e)
|
||||
elif code is not None:
|
||||
raise MastodonIllegalArgumentError('Invalid access token or redirect_uris: %s' % e)
|
||||
else:
|
||||
raise MastodonIllegalArgumentError('Invalid request: %s' % e)
|
||||
|
||||
requested_scopes = " ".join(sorted(scopes))
|
||||
received_scopes = " ".join(sorted(response["scope"].split(" ")))
|
||||
|
@ -372,7 +384,7 @@ class Mastodon:
|
|||
|
||||
Returns a list of user dicts.
|
||||
"""
|
||||
return self.__api_request('GET', '/api/v1/accounts/' + str(id) + '/following')
|
||||
return self.__api_request('GET', '/api/v1/accounts/' + str(id) + '/following',do_fetch_all=True)
|
||||
|
||||
def account_followers(self, id):
|
||||
"""
|
||||
|
@ -709,7 +721,7 @@ class Mastodon:
|
|||
|
||||
return (date_time_utc - epoch_utc).total_seconds()
|
||||
|
||||
def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True):
|
||||
def __api_request(self, method, endpoint, params={}, files={}, do_ratelimiting=True, get_r_object=False):
|
||||
"""
|
||||
Internal API request helper.
|
||||
"""
|
||||
|
@ -736,10 +748,10 @@ class Mastodon:
|
|||
time.sleep(to_next)
|
||||
|
||||
# Generate request headers
|
||||
if self.access_token != None:
|
||||
if self.access_token is not None:
|
||||
headers = {'Authorization': 'Bearer ' + self.access_token}
|
||||
|
||||
if self.debug_requests == True:
|
||||
if self.debug_requests is True:
|
||||
print('Mastodon: Request to endpoint "' + endpoint + '" using method "' + method + '".')
|
||||
print('Parameters: ' + str(params))
|
||||
print('Headers: ' + str(headers))
|
||||
|
@ -772,7 +784,7 @@ class Mastodon:
|
|||
if self.debug_requests == True:
|
||||
print('Mastodon: Response received with code ' + str(response_object.status_code) + '.')
|
||||
print('response headers: ' + str(response_object.headers))
|
||||
print('Response text content: ' + str(response_object.text))
|
||||
#print('Response text content: ' + str(response_object.text))
|
||||
|
||||
if response_object.status_code == 404:
|
||||
raise MastodonAPIError('Endpoint not found.')
|
||||
|
@ -781,7 +793,14 @@ class Mastodon:
|
|||
raise MastodonAPIError('General API problem.')
|
||||
|
||||
try:
|
||||
response = response_object.json()
|
||||
if get_r_object:
|
||||
response = ResponseObject._load(response_object, method, params, files, do_ratelimiting, self.api_base_url)
|
||||
else:
|
||||
temp_r = ResponseObject(response_object.json(), response_object, method, params, files, do_ratelimiting, self.api_base_url)
|
||||
if temp_r is not None:
|
||||
response = temp_r.response
|
||||
else:
|
||||
print("Big error")
|
||||
except:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
@ -815,9 +834,9 @@ class Mastodon:
|
|||
to_next = self.ratelimit_reset - time.time()
|
||||
if to_next > 0:
|
||||
# As a precaution, never sleep longer than 5 minutes
|
||||
request_complete = False
|
||||
to_next = min(to_next, 5 * 60)
|
||||
time.sleep(to_next)
|
||||
request_complete = False
|
||||
|
||||
return response
|
||||
|
||||
|
@ -859,22 +878,3 @@ class Mastodon:
|
|||
del params[key]
|
||||
|
||||
return params
|
||||
|
||||
##
|
||||
# Exceptions
|
||||
##
|
||||
class MastodonIllegalArgumentError(ValueError):
|
||||
pass
|
||||
|
||||
class MastodonFileNotFoundError(IOError):
|
||||
pass
|
||||
|
||||
class MastodonNetworkError(IOError):
|
||||
pass
|
||||
|
||||
class MastodonAPIError(Exception):
|
||||
pass
|
||||
|
||||
class MastodonRatelimitError(Exception):
|
||||
pass
|
||||
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
from mastodon.Mastodon import Mastodon
|
||||
from mastodon.streaming import StreamListener, MalformedEventError
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
__all__ = ['Mastodon', 'StreamListener', 'MalformedEventError']
|
||||
from mastodon.Mastodon import Mastodon
|
||||
from mastodon.exceptions import *
|
||||
from mastodon.response import ResponseObject
|
||||
from mastodon.utils import *
|
||||
from mastodon.streaming import StreamListener, MalformedEventError
|
||||
|
|
19
mastodon/exceptions.py
Normal file
19
mastodon/exceptions.py
Normal file
|
@ -0,0 +1,19 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""Exceptions"""
|
||||
|
||||
class MastodonIllegalArgumentError(ValueError):
|
||||
pass
|
||||
|
||||
class MastodonFileNotFoundError(IOError):
|
||||
pass
|
||||
|
||||
class MastodonNetworkError(IOError):
|
||||
pass
|
||||
|
||||
class MastodonAPIError(Exception):
|
||||
pass
|
||||
|
||||
class MastodonRatelimitError(Exception):
|
||||
pass
|
100
mastodon/response.py
Normal file
100
mastodon/response.py
Normal file
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import json
|
||||
|
||||
import requests
|
||||
|
||||
from mastodon.exceptions import *
|
||||
import Mastodon
|
||||
|
||||
|
||||
class ResponseObject(object):
|
||||
def __init__(self, response: dict, response_object: requests.models.Response, method: str, params: dict, files: dict, do_ratelimiting: bool, api_base_url: str):
|
||||
self._response = None
|
||||
self.response = response
|
||||
self.response_object = response_object
|
||||
self.method = method
|
||||
self.params = params
|
||||
self.files = files
|
||||
self.do_ratelimiting = do_ratelimiting
|
||||
self.api_base_url = api_base_url
|
||||
self.temp_m = Mastodon("temp")
|
||||
|
||||
|
||||
@classmethod
|
||||
def _load(cls, response: requests.models.Response, method: str = "GET", params: dict = {}, files = {}, do_ratelimiting = True, api_base_url: str = 'https://mastodon.social'):
|
||||
if type(response) is requests.models.Response:
|
||||
try:
|
||||
r = response.json()
|
||||
print("worked")
|
||||
return cls(r, response, method, params, files, do_ratelimiting, api_base_url)
|
||||
except Exception:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def __iter__(self):
|
||||
try:
|
||||
while True:
|
||||
if 'Link' in self.response_object.headers:
|
||||
tmp_url = requests.utils.parse_header_links(self.response_object.headers['Link'].rstrip('>').replace('>,<', ',<'))
|
||||
next_url = None
|
||||
if tmp_url:
|
||||
for url in tmp_url:
|
||||
if url['rel'] == 'next':
|
||||
next_url = url['url'].replace(self.api_base_url,'')
|
||||
break
|
||||
if next_url is not None:
|
||||
tmp_response = self.temp_m.__api_request(self.method, next_url, params=self.params, files=self.files, do_ratelimiting=self.do_ratelimiting)
|
||||
if type(tmp_response) is dict:
|
||||
self.response = tmp_response
|
||||
yield tmp_response
|
||||
else:
|
||||
return
|
||||
else:
|
||||
self.response = self.response_object.json()
|
||||
return self.response_object.json()
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return '<ResponseObject [%s]>' % (self.response_object.url)
|
||||
|
||||
def __str__(self):
|
||||
if self.response is None:
|
||||
return ""
|
||||
elif type(self.response) is str:
|
||||
return self.response
|
||||
elif type(self.response) is list:
|
||||
return "\n".join(self.response)
|
||||
elif type(self.response) is dict:
|
||||
return json.dumps(self.response)
|
||||
else:
|
||||
return str(self.response)
|
||||
|
||||
def __dict__(self):
|
||||
if type(self.response) is not dict:
|
||||
return {"response": None}
|
||||
else:
|
||||
return {"response": self.response}
|
||||
|
||||
@property
|
||||
def response(self) -> dict:
|
||||
return self._response
|
||||
|
||||
@response.setter
|
||||
def response(self, value: dict):
|
||||
self._response = value
|
||||
return
|
||||
|
||||
def fetch_all(self):
|
||||
r = []
|
||||
for page in self:
|
||||
if page is not None:
|
||||
r.append(page)
|
||||
else:
|
||||
return self.response
|
||||
return r
|
184
mastodon/utils.py
Normal file
184
mastodon/utils.py
Normal file
|
@ -0,0 +1,184 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import time
|
||||
import datetime
|
||||
from contextlib import closing
|
||||
|
||||
import pytz
|
||||
import dateutil
|
||||
import requests
|
||||
|
||||
from mastodon.exceptions import *
|
||||
|
||||
|
||||
"""Internal helpers, dragons probably"""
|
||||
|
||||
def __datetime_to_epoch(self, date_time):
|
||||
"""
|
||||
Converts a python datetime to unix epoch, accounting for
|
||||
time zones and such.
|
||||
|
||||
Assumes UTC if timezone is not given.
|
||||
"""
|
||||
date_time_utc = None
|
||||
if date_time.tzinfo == None:
|
||||
date_time_utc = date_time.replace(tzinfo = pytz.utc)
|
||||
else:
|
||||
date_time_utc = date_time.astimezone(pytz.utc)
|
||||
|
||||
epoch_utc = datetime.datetime.utcfromtimestamp(0).replace(tzinfo = pytz.utc)
|
||||
|
||||
return (date_time_utc - epoch_utc).total_seconds()
|
||||
|
||||
def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True):
|
||||
"""
|
||||
Internal API request helper.
|
||||
"""
|
||||
response = None
|
||||
headers = None
|
||||
next_url = None
|
||||
|
||||
# "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it
|
||||
# would take to not hit the rate limit at that request rate.
|
||||
if do_ratelimiting and self.ratelimit_method == "pace":
|
||||
if self.ratelimit_remaining == 0:
|
||||
to_next = self.ratelimit_reset - time.time()
|
||||
if to_next > 0:
|
||||
# As a precaution, never sleep longer than 5 minutes
|
||||
to_next = min(to_next, 5 * 60)
|
||||
time.sleep(to_next)
|
||||
else:
|
||||
time_waited = time.time() - self.ratelimit_lastcall
|
||||
time_wait = float(self.ratelimit_reset - time.time()) / float(self.ratelimit_remaining)
|
||||
remaining_wait = time_wait - time_waited
|
||||
|
||||
if remaining_wait > 0:
|
||||
to_next = remaining_wait / self.ratelimit_pacefactor
|
||||
to_next = min(to_next, 5 * 60)
|
||||
time.sleep(to_next)
|
||||
|
||||
# Generate request headers
|
||||
if self.access_token is not None:
|
||||
headers = {'Authorization': 'Bearer ' + self.access_token}
|
||||
|
||||
if self.debug_requests is True:
|
||||
print('Mastodon: Request to endpoint "' + endpoint + '" using method "' + method + '".')
|
||||
print('Parameters: ' + str(params))
|
||||
print('Headers: ' + str(headers))
|
||||
print('Files: ' + str(files))
|
||||
|
||||
# Make request
|
||||
request_complete = False
|
||||
while not request_complete:
|
||||
request_complete = True
|
||||
|
||||
response_object = None
|
||||
try:
|
||||
if method == 'GET':
|
||||
response_object = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout)
|
||||
|
||||
if method == 'POST':
|
||||
response_object = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout)
|
||||
|
||||
if method == 'DELETE':
|
||||
response_object = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files, timeout = self.request_timeout)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise MastodonNetworkError("Could not complete request: %s" % e)
|
||||
|
||||
if response_object == None:
|
||||
raise MastodonIllegalArgumentError("Illegal request.")
|
||||
|
||||
# Handle response
|
||||
if self.debug_requests == True:
|
||||
print('Mastodon: Response received with code ' + str(response_object.status_code) + '.')
|
||||
print('response headers: ' + str(response_object.headers))
|
||||
#print('Response text content: ' + str(response_object.text))
|
||||
|
||||
if response_object.status_code == 404:
|
||||
raise MastodonAPIError('Endpoint not found.')
|
||||
|
||||
if response_object.status_code == 500:
|
||||
raise MastodonAPIError('General API problem.')
|
||||
|
||||
try:
|
||||
response = response_object.json()
|
||||
except:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise MastodonAPIError("Could not parse response as JSON, response code was %s, bad json content was '%s'" % (response_object.status_code, response_object.content))
|
||||
|
||||
# Handle rate limiting
|
||||
if 'X-RateLimit-Remaining' in response_object.headers and do_ratelimiting:
|
||||
self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining'])
|
||||
self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit'])
|
||||
|
||||
try:
|
||||
ratelimit_reset_datetime = dateutil.parser.parse(response_object.headers['X-RateLimit-Reset'])
|
||||
self.ratelimit_reset = self.__datetime_to_epoch(ratelimit_reset_datetime)
|
||||
|
||||
# Adjust server time to local clock
|
||||
server_time_datetime = dateutil.parser.parse(response_object.headers['Date'])
|
||||
server_time = self.__datetime_to_epoch(server_time_datetime)
|
||||
server_time_diff = time.time() - server_time
|
||||
self.ratelimit_reset += server_time_diff
|
||||
self.ratelimit_lastcall = time.time()
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise MastodonRatelimitError("Rate limit time calculations failed: %s" % e)
|
||||
|
||||
if "error" in response and response["error"] == "Throttled":
|
||||
if self.ratelimit_method == "throw":
|
||||
raise MastodonRatelimitError("Hit rate limit.")
|
||||
|
||||
if self.ratelimit_method == "wait" or self.ratelimit_method == "pace":
|
||||
to_next = self.ratelimit_reset - time.time()
|
||||
if to_next > 0:
|
||||
# As a precaution, never sleep longer than 5 minutes
|
||||
request_complete = False
|
||||
to_next = min(to_next, 5 * 60)
|
||||
time.sleep(to_next)
|
||||
|
||||
return response
|
||||
|
||||
def __stream(self, endpoint, listener, params = {}):
|
||||
"""
|
||||
Internal streaming API helper.
|
||||
"""
|
||||
|
||||
headers = {}
|
||||
if self.access_token != None:
|
||||
headers = {'Authorization': 'Bearer ' + self.access_token}
|
||||
|
||||
url = self.api_base_url + endpoint
|
||||
with closing(requests.get(url, headers = headers, data = params, stream = True)) as r:
|
||||
listener.handle_stream(r.iter_lines())
|
||||
|
||||
|
||||
def __generate_params(self, params, exclude = []):
|
||||
"""
|
||||
Internal named-parameters-to-dict helper.
|
||||
|
||||
Note for developers: If called with locals() as params,
|
||||
as is the usual practice in this code, the __generate_params call
|
||||
(or at least the locals() call) should generally be the first thing
|
||||
in your function.
|
||||
"""
|
||||
params = dict(params)
|
||||
|
||||
del params['self']
|
||||
param_keys = list(params.keys())
|
||||
for key in param_keys:
|
||||
if params[key] == None or key in exclude:
|
||||
del params[key]
|
||||
|
||||
param_keys = list(params.keys())
|
||||
for key in param_keys:
|
||||
if isinstance(params[key], list):
|
||||
params[key + "[]"] = params[key]
|
||||
del params[key]
|
||||
|
||||
return params
|
Loading…
Referencia en una nova incidència