aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLorenz Diener <[email protected]>2016-11-25 18:17:39 +0100
committerLorenz Diener <[email protected]>2016-11-25 18:17:39 +0100
commite4e3a8eb93721bf12e4a5b2bf45dc3c2a473dc6f (patch)
tree09947f42aff122f0b98307fbd12afc08c6432091 /mastodon
parent2729ca19319edcc7a0d8df3e211010ccf857b211 (diff)
downloadmastodon.py-e4e3a8eb93721bf12e4a5b2bf45dc3c2a473dc6f.tar.gz
Ratelimit code
Diffstat (limited to 'mastodon')
-rw-r--r--mastodon/Mastodon.py154
1 files changed, 124 insertions, 30 deletions
diff --git a/mastodon/Mastodon.py b/mastodon/Mastodon.py
index 7bd6473..8af739d 100644
--- a/mastodon/Mastodon.py
+++ b/mastodon/Mastodon.py
@@ -7,6 +7,7 @@ import mimetypes
7import time 7import time
8import random 8import random
9import string 9import string
10from datetime import datetime
10 11
11class Mastodon: 12class Mastodon:
12 """ 13 """
@@ -21,6 +22,7 @@ class Mastodon:
21 """ 22 """
22 __DEFAULT_BASE_URL = 'https://mastodon.social' 23 __DEFAULT_BASE_URL = 'https://mastodon.social'
23 24
25
24 ### 26 ###
25 # Registering apps 27 # Registering apps
26 ### 28 ###
@@ -32,6 +34,9 @@ class Mastodon:
32 Specify redirect_uris if you want users to be redirected to a certain page after authenticating. 34 Specify redirect_uris if you want users to be redirected to a certain page after authenticating.
33 Specify to_file to persist your apps info to a file so you can use them in the constructor. 35 Specify to_file to persist your apps info to a file so you can use them in the constructor.
34 Specify api_base_url if you want to register an app on an instance different from the flagship one. 36 Specify api_base_url if you want to register an app on an instance different from the flagship one.
37
38 Presently, app registration is open by default, but this is not guaranteed to be the case for all
39 future mastodon instances or even the flagship instance in the future.
35 40
36 Returns client_id and client_secret. 41 Returns client_id and client_secret.
37 """ 42 """
@@ -57,13 +62,22 @@ class Mastodon:
57 ### 62 ###
58 # Authentication, including constructor 63 # Authentication, including constructor
59 ### 64 ###
60 def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False): 65 def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False, ratelimit_method = "wait", ratelimit_pacefactor = 0.9):
61 """ 66 """
62 Creates a new API wrapper instance based on the given client_secret and client_id. If you 67 Creates a new API wrapper instance based on the given client_secret and client_id. If you
63 give a client_id and it is not a file, you must also give a secret. 68 give a client_id and it is not a file, you must also give a secret.
64 69
65 You can also directly specify an access_token, directly or as a file. 70 You can also directly specify an access_token, directly or as a file.
66 71
72 Mastodon.py can try to respect rate limits in several ways, controlled by ratelimit_method.
73 "throw" makes functions throw a MastodonRatelimitError when the rate
74 limit is hit. "wait" mode will, once the limit is hit, wait and retry the request as soon
75 as the rate limit resets, until it succeeds. "pace" works like throw, but tries to wait in
76 between calls so that the limit is generally not hit (How hard it tries to not hit the rate
77 limit can be controlled by ratelimit_pacefactor). The default setting is "wait". Note that
78 even in "wait" and "pace" mode, requests can still fail due to network or other problems! Also
79 note that "pace" and "wait" are NOT thread safe.
80
67 Specify api_base_url if you wish to talk to an instance other than the flagship one. 81 Specify api_base_url if you wish to talk to an instance other than the flagship one.
68 If a file is given as client_id, read client ID and secret from that file 82 If a file is given as client_id, read client ID and secret from that file
69 """ 83 """
@@ -72,6 +86,13 @@ class Mastodon:
72 self.client_secret = client_secret 86 self.client_secret = client_secret
73 self.access_token = access_token 87 self.access_token = access_token
74 self.debug_requests = debug_requests 88 self.debug_requests = debug_requests
89 self.ratelimit_method = ratelimit_method
90
91 self.ratelimit_limit = 150
92 self.ratelimit_reset = time.time()
93 self.ratelimit_remaining = 150
94 self.ratelimit_lastcall = time.time()
95 self.ratelimit_pacefactor = 0.9
75 96
76 if os.path.isfile(self.client_id): 97 if os.path.isfile(self.client_id):
77 with open(self.client_id, 'r') as secret_file: 98 with open(self.client_id, 'r') as secret_file:
@@ -79,7 +100,7 @@ class Mastodon:
79 self.client_secret = secret_file.readline().rstrip() 100 self.client_secret = secret_file.readline().rstrip()
80 else: 101 else:
81 if self.client_secret == None: 102 if self.client_secret == None:
82 raise ValueError('Specified client id directly, but did not supply secret') 103 raise MastodonIllegalArgumentError('Specified client id directly, but did not supply secret')
83 104
84 if self.access_token != None and os.path.isfile(self.access_token): 105 if self.access_token != None and os.path.isfile(self.access_token):
85 with open(self.access_token, 'r') as token_file: 106 with open(self.access_token, 'r') as token_file:
@@ -87,8 +108,10 @@ class Mastodon:
87 108
88 def log_in(self, username, password, scopes = ['read', 'write', 'follow'], to_file = None): 109 def log_in(self, username, password, scopes = ['read', 'write', 'follow'], to_file = None):
89 """ 110 """
90 Logs in and sets access_token to what was returned. 111 Logs in and sets access_token to what was returned. Note that your
91 Can persist access token to file. 112 username is the e-mail you use to log in into mastodon.
113
114 Can persist access token to file, to be used in the constructor.
92 115
93 Will throw an exception if username / password are wrong, scopes are not 116 Will throw an exception if username / password are wrong, scopes are not
94 valid or granted scopes differ from requested. 117 valid or granted scopes differ from requested.
@@ -105,13 +128,13 @@ class Mastodon:
105 response = self.__api_request('POST', '/oauth/token', params) 128 response = self.__api_request('POST', '/oauth/token', params)
106 self.access_token = response['access_token'] 129 self.access_token = response['access_token']
107 except: 130 except:
108 raise ValueError('Invalid user name, password or scopes.') 131 raise MastodonIllegalArgumentError('Invalid user name, password or scopes.')
109 132
110 requested_scopes = " ".join(sorted(scopes)) 133 requested_scopes = " ".join(sorted(scopes))
111 received_scopes = " ".join(sorted(response["scope"].split(" "))) 134 received_scopes = " ".join(sorted(response["scope"].split(" ")))
112 135
113 if requested_scopes != received_scopes: 136 if requested_scopes != received_scopes:
114 raise ValueError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".') 137 raise MastodonAPIError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".')
115 138
116 if to_file != None: 139 if to_file != None:
117 with open(to_file, 'w') as token_file: 140 with open(to_file, 'w') as token_file:
@@ -352,8 +375,8 @@ class Mastodon:
352 the ID that can then be used in status_post() to attach the media to 375 the ID that can then be used in status_post() to attach the media to
353 a toot. 376 a toot.
354 377
355 Throws a ValueError if the mime type of the passed data or file can 378 Throws a MastodonIllegalArgumentError if the mime type of the
356 not be determined properly. 379 passed data or file can not be determined properly.
357 """ 380 """
358 381
359 if os.path.isfile(media_file): 382 if os.path.isfile(media_file):
@@ -361,7 +384,7 @@ class Mastodon:
361 media_file = open(media_file, 'rb') 384 media_file = open(media_file, 'rb')
362 385
363 if mime_type == None: 386 if mime_type == None:
364 raise ValueError('Could not determine mime type or data passed directly without mime type.') 387 raise MastodonIllegalArgumentError('Could not determine mime type or data passed directly without mime type.')
365 388
366 random_suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) 389 random_suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
367 file_name = "mastodonpyupload_" + str(time.time()) + "_" + str(random_suffix) + mimetypes.guess_extension(mime_type) 390 file_name = "mastodonpyupload_" + str(time.time()) + "_" + str(random_suffix) + mimetypes.guess_extension(mime_type)
@@ -375,11 +398,32 @@ class Mastodon:
375 def __api_request(self, method, endpoint, params = {}, files = {}): 398 def __api_request(self, method, endpoint, params = {}, files = {}):
376 """ 399 """
377 Internal API request helper. 400 Internal API request helper.
401
402 TODO FIXME: time.time() does not match server time neccesarily. Using the time from the request
403 would be correct.
404
405 TODO FIXME: Date parsing can fail. Should probably use a proper "date parsing" module rather than
406 rely on the server to return the right thing.
378 """ 407 """
379 response = None 408 response = None
380 headers = None 409 headers = None
381 410
382 411 # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it
412 # would take to not hit the rate limit at that request rate.
413 if self.ratelimit_method == "pace":
414 if self.ratelimit_remaining == 0:
415 to_next = self.ratelimit_reset - time.time()
416 if to_next > 0:
417 time.sleep(to_next)
418 else:
419 time_waited = time.time() - self.ratelimit_lastcall
420 time_wait = float(self.ratelimit_reset - time.time()) / float(self.ratelimit_remaining)
421 remaining_wait = time_wait - time_waited
422
423 if remaining_wait > 0:
424 time.sleep(remaining_wait * self.ratelimit_pacefactor)
425
426 # Generate request headers
383 if self.access_token != None: 427 if self.access_token != None:
384 headers = {'Authorization': 'Bearer ' + self.access_token} 428 headers = {'Authorization': 'Bearer ' + self.access_token}
385 429
@@ -389,26 +433,60 @@ class Mastodon:
389 print('Headers: ' + str(headers)) 433 print('Headers: ' + str(headers))
390 print('Files: ' + str(files)) 434 print('Files: ' + str(files))
391 435
392 if method == 'GET': 436 # Make request
393 response = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files) 437 request_complete = False
394 438 while not request_complete:
395 if method == 'POST': 439 request_complete = True
396 response = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
397 440
398 if method == 'DELETE': 441 response_object = None
399 response = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files) 442 try:
400 443 if method == 'GET':
401 if response.status_code == 404: 444 response_object = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files)
402 raise IOError('Endpoint not found.') 445
403 446 if method == 'POST':
404 if response.status_code == 500: 447 response_object = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
405 raise IOError('General API problem.') 448
406 449 if method == 'DELETE':
407 try: 450 response_object = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files)
408 response = response.json() 451 except:
409 except: 452 raise MastodonNetworkError("Could not complete request.")
410 raise ValueError("Could not parse response as JSON, respose code was " + str(response.status_code)) 453
411 454 if response_object == None:
455 raise MastodonIllegalArgumentError("Illegal request.")
456
457 # Handle response
458 if self.debug_requests == True:
459 print('Mastodon: Response received with code ' + str(response_object.status_code) + '.')
460 print('Respose headers: ' + str(response_object.headers))
461 print('Response text content: ' + str(response_object.text))
462
463 if response_object.status_code == 404:
464 raise MastodonAPIError('Endpoint not found.')
465
466 if response_object.status_code == 500:
467 raise MastodonAPIError('General API problem.')
468
469 try:
470 response = response_object.json()
471 except:
472 raise MastodonAPIError("Could not parse response as JSON, respose code was " + str(response_object.status_code))
473
474 # Handle rate limiting
475 self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining'])
476 self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit'])
477 self.ratelimit_reset = (datetime.strptime(response_object.headers['X-RateLimit-Reset'], "%Y-%m-%dT%H:%M:%S.%fZ") - datetime(1970, 1, 1)).total_seconds()
478 self.ratelimit_lastcall = time.time()
479
480 if "error" in response and response["error"] == "Throttled":
481 if self.ratelimit_method == "throw":
482 raise MastodonRatelimitError("Hit rate limit.")
483
484 if self.ratelimit_method == "wait" or self.ratelimit_method == "pace":
485 to_next = self.ratelimit_reset - time.time()
486 if to_next > 0:
487 time.sleep(to_next)
488 request_complete = False
489
412 return response 490 return response
413 491
414 def __generate_params(self, params, exclude = []): 492 def __generate_params(self, params, exclude = []):
@@ -430,3 +508,19 @@ class Mastodon:
430 del params[key] 508 del params[key]
431 509
432 return params 510 return params
511
512##
513# Exceptions
514##
515class MastodonIllegalArgumentError(ValueError):
516 pass
517
518class MastodonNetworkError(IOError):
519 pass
520
521class MastodonAPIError(Exception):
522 pass
523
524class MastodonRatelimitError(Exception):
525 pass
526
Powered by cgit v1.2.3 (git 2.41.0)