Ratelimit code

mastodon/Mastodon.py

@@ -7,6 +7,7 @@ import mimetypes
 import time
 import random
 import string
+from datetime import datetime
 
 class Mastodon:
     """ 
@@ -21,6 +22,7 @@ class Mastodon:
     """
     __DEFAULT_BASE_URL = 'https://mastodon.social'
     
+    
     ###
     # Registering apps
     ###
@@ -33,6 +35,9 @@ class Mastodon:
         Specify to_file to persist your apps info to a file so you can use them in the constructor.
         Specify api_base_url if you want to register an app on an instance different from the flagship one.
         
+        Presently, app registration is open by default, but this is not guaranteed to be the case for all
+        future mastodon instances or even the flagship instance in the future.
+           
         Returns client_id and client_secret.
         """
         request_data = {
@@ -57,13 +62,22 @@ class Mastodon:
     ###
     # Authentication, including constructor
     ###
-    def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False):
+    def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False, ratelimit_method = "wait", ratelimit_pacefactor = 0.9):
         """
         Creates a new API wrapper instance based on the given client_secret and client_id. If you
         give a client_id and it is not a file, you must also give a secret.
            
         You can also directly specify an access_token, directly or as a file.
         
+        Mastodon.py can try to respect rate limits in several ways, controlled by ratelimit_method.
+        "throw" makes functions throw a MastodonRatelimitError when the rate
+        limit is hit. "wait" mode will, once the limit is hit, wait and retry the request as soon
+        as the rate limit resets, until it succeeds. "pace" works like throw, but tries to wait in
+        between calls so that the limit is generally not hit (How hard it tries to not hit the rate 
+        limit can be controlled by ratelimit_pacefactor). The default setting is "wait". Note that
+        even in "wait" and "pace" mode, requests can still fail due to network or other problems! Also
+        note that "pace" and "wait" are NOT thread safe.
+        
         Specify api_base_url if you wish to talk to an instance other than the flagship one.
         If a file is given as client_id, read client ID and secret from that file
         """
@@ -72,6 +86,13 @@ class Mastodon:
         self.client_secret = client_secret
         self.access_token = access_token
         self.debug_requests = debug_requests
+        self.ratelimit_method = ratelimit_method
+        
+        self.ratelimit_limit = 150
+        self.ratelimit_reset = time.time()
+        self.ratelimit_remaining = 150
+        self.ratelimit_lastcall = time.time()
+        self.ratelimit_pacefactor = 0.9
         
         if os.path.isfile(self.client_id):
             with open(self.client_id, 'r') as secret_file:
@@ -79,7 +100,7 @@ class Mastodon:
                 self.client_secret = secret_file.readline().rstrip()
         else:
             if self.client_secret == None:
-                raise ValueError('Specified client id directly, but did not supply secret')
+                raise MastodonIllegalArgumentError('Specified client id directly, but did not supply secret')
                 
         if self.access_token != None and os.path.isfile(self.access_token):
             with open(self.access_token, 'r') as token_file:
@@ -87,8 +108,10 @@ class Mastodon:
                 
     def log_in(self, username, password, scopes = ['read', 'write', 'follow'], to_file = None):
         """
-        Logs in and sets access_token to what was returned.
+        Logs in and sets access_token to what was returned. Note that your
-        Can persist access token to file.
+        username is the e-mail you use to log in into mastodon.
+        
+        Can persist access token to file, to be used in the constructor.
         
         Will throw an exception if username / password are wrong, scopes are not
         valid or granted scopes differ from requested.
@@ -105,13 +128,13 @@ class Mastodon:
             response = self.__api_request('POST', '/oauth/token', params)      
             self.access_token = response['access_token']
         except:
-            raise ValueError('Invalid user name, password or scopes.')
+            raise MastodonIllegalArgumentError('Invalid user name, password or scopes.')
         
         requested_scopes = " ".join(sorted(scopes))
         received_scopes = " ".join(sorted(response["scope"].split(" ")))
         
         if requested_scopes != received_scopes:
-            raise ValueError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".')
+            raise MastodonAPIError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".')
         
         if to_file != None:
             with open(to_file, 'w') as token_file:
@@ -352,8 +375,8 @@ class Mastodon:
         the ID that can then be used in status_post() to attach the media to
         a toot.
         
-        Throws a ValueError if the mime type of the passed data or file can
+        Throws a MastodonIllegalArgumentError if the mime type of the 
-        not be determined properly.
+        passed data or file can not be determined properly.
         """
         
         if os.path.isfile(media_file):
@@ -361,7 +384,7 @@ class Mastodon:
             media_file = open(media_file, 'rb')
             
         if mime_type == None:
-            raise ValueError('Could not determine mime type or data passed directly without mime type.')
+            raise MastodonIllegalArgumentError('Could not determine mime type or data passed directly without mime type.')
         
         random_suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
         file_name = "mastodonpyupload_" + str(time.time()) + "_" + str(random_suffix) + mimetypes.guess_extension(mime_type)
@@ -375,11 +398,32 @@ class Mastodon:
     def __api_request(self, method, endpoint, params = {}, files = {}):
         """
         Internal API request helper.
+        
+        TODO FIXME: time.time() does not match server time neccesarily. Using the time from the request
+        would be correct.
+        
+        TODO FIXME: Date parsing can fail. Should probably use a proper "date parsing" module rather than
+        rely on the server to return the right thing.
         """
         response = None
         headers = None
         
+        # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it
+        # would take to not hit the rate limit at that request rate.
+        if self.ratelimit_method == "pace":
+            if self.ratelimit_remaining == 0:
+                to_next = self.ratelimit_reset - time.time()
+                if to_next > 0:
+                    time.sleep(to_next)
+            else:
+                time_waited = time.time() - self.ratelimit_lastcall
+                time_wait = float(self.ratelimit_reset - time.time()) / float(self.ratelimit_remaining)
+                remaining_wait = time_wait - time_waited
             
+            if remaining_wait > 0:
+                time.sleep(remaining_wait * self.ratelimit_pacefactor)
+                
+        # Generate request headers
         if self.access_token != None:
             headers = {'Authorization': 'Bearer ' + self.access_token}
         
@@ -389,25 +433,59 @@ class Mastodon:
             print('Headers: ' + str(headers))
             print('Files: ' + str(files))
 
-        if method == 'GET':
+        # Make request
-            response = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files)
+        request_complete = False
+        while not request_complete:
+            request_complete = True
             
-        if method == 'POST':
+            response_object = None
-            response = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
+            try:
+                if method == 'GET':
+                    response_object = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files)
                 
-        if method == 'DELETE':
+                if method == 'POST':
-            response = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files)
+                    response_object = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
                     
-        if response.status_code == 404:
+                if method == 'DELETE':
-            raise IOError('Endpoint not found.')
+                    response_object = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files)
+            except:
+                raise MastodonNetworkError("Could not complete request.")
         
-        if response.status_code == 500:
+            if response_object == None:
-            raise IOError('General API problem.')
+                raise MastodonIllegalArgumentError("Illegal request.")
             
-        try:
+            # Handle response
-            response = response.json()
+            if self.debug_requests == True:
-        except:
+                print('Mastodon: Response received with code ' + str(response_object.status_code) + '.')
-            raise ValueError("Could not parse response as JSON, respose code was " + str(response.status_code))
+                print('Respose headers: ' + str(response_object.headers))
+                print('Response text content: ' + str(response_object.text))
+            
+            if response_object.status_code == 404:
+                raise MastodonAPIError('Endpoint not found.')
+            
+            if response_object.status_code == 500:
+                raise MastodonAPIError('General API problem.')
+            
+            try:
+                response = response_object.json()
+            except:
+                raise MastodonAPIError("Could not parse response as JSON, respose code was " + str(response_object.status_code))
+        
+            # Handle rate limiting
+            self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining'])
+            self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit'])
+            self.ratelimit_reset = (datetime.strptime(response_object.headers['X-RateLimit-Reset'], "%Y-%m-%dT%H:%M:%S.%fZ") - datetime(1970, 1, 1)).total_seconds()
+            self.ratelimit_lastcall = time.time()
+            
+            if "error" in response and response["error"] == "Throttled":
+                if self.ratelimit_method == "throw":
+                    raise MastodonRatelimitError("Hit rate limit.")
+                
+                if self.ratelimit_method == "wait" or self.ratelimit_method == "pace":
+                    to_next = self.ratelimit_reset - time.time()
+                    if to_next > 0:
+                        time.sleep(to_next)
+                    request_complete = False
                     
         return response
     
@@ -430,3 +508,19 @@ class Mastodon:
                 del params[key]
                 
         return params
+
+##
+# Exceptions
+##
+class MastodonIllegalArgumentError(ValueError):
+    pass
+
+class MastodonNetworkError(IOError):
+    pass
+
+class MastodonAPIError(Exception):
+    pass
+
+class MastodonRatelimitError(Exception):
+    pass
+