aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLorenz Diener <[email protected]>2016-11-25 23:18:51 +0100
committerGitHub <[email protected]>2016-11-25 23:18:51 +0100
commit3ce225dbeff8dc81539108cad376c0ab7db2125f (patch)
tree8c8994ea4021902b329f8c79a4318f691cbbad36 /mastodon
parentb958ce54ba32968ef159bda91c8f480c74374f68 (diff)
parent61775d90831704d012b9f3d6c5453ca738bc0724 (diff)
downloadmastodon.py-3ce225dbeff8dc81539108cad376c0ab7db2125f.tar.gz
Merge pull request #3 from halcy/ratelimits
Implement ratelimiting
Diffstat (limited to 'mastodon')
-rw-r--r--mastodon/Mastodon.py202
1 files changed, 165 insertions, 37 deletions
diff --git a/mastodon/Mastodon.py b/mastodon/Mastodon.py
index eb13859..bb16d95 100644
--- a/mastodon/Mastodon.py
+++ b/mastodon/Mastodon.py
@@ -7,6 +7,10 @@ import mimetypes
7import time 7import time
8import random 8import random
9import string 9import string
10import pytz
11import datetime
12import dateutil
13import dateutil.parser
10 14
11class Mastodon: 15class Mastodon:
12 """ 16 """
@@ -21,6 +25,7 @@ class Mastodon:
21 """ 25 """
22 __DEFAULT_BASE_URL = 'https://mastodon.social' 26 __DEFAULT_BASE_URL = 'https://mastodon.social'
23 27
28
24 ### 29 ###
25 # Registering apps 30 # Registering apps
26 ### 31 ###
@@ -32,6 +37,9 @@ class Mastodon:
32 Specify redirect_uris if you want users to be redirected to a certain page after authenticating. 37 Specify redirect_uris if you want users to be redirected to a certain page after authenticating.
33 Specify to_file to persist your apps info to a file so you can use them in the constructor. 38 Specify to_file to persist your apps info to a file so you can use them in the constructor.
34 Specify api_base_url if you want to register an app on an instance different from the flagship one. 39 Specify api_base_url if you want to register an app on an instance different from the flagship one.
40
41 Presently, app registration is open by default, but this is not guaranteed to be the case for all
42 future mastodon instances or even the flagship instance in the future.
35 43
36 Returns client_id and client_secret. 44 Returns client_id and client_secret.
37 """ 45 """
@@ -57,13 +65,22 @@ class Mastodon:
57 ### 65 ###
58 # Authentication, including constructor 66 # Authentication, including constructor
59 ### 67 ###
60 def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False): 68 def __init__(self, client_id, client_secret = None, access_token = None, api_base_url = __DEFAULT_BASE_URL, debug_requests = False, ratelimit_method = "wait", ratelimit_pacefactor = 1.1):
61 """ 69 """
62 Create a new API wrapper instance based on the given client_secret and client_id. If you 70 Create a new API wrapper instance based on the given client_secret and client_id. If you
63 give a client_id and it is not a file, you must also give a secret. 71 give a client_id and it is not a file, you must also give a secret.
64 72
65 You can also directly specify an access_token, directly or as a file. 73 You can also specify an access_token, directly or as a file (as written by log_in).
66 74
75 Mastodon.py can try to respect rate limits in several ways, controlled by ratelimit_method.
76 "throw" makes functions throw a MastodonRatelimitError when the rate
77 limit is hit. "wait" mode will, once the limit is hit, wait and retry the request as soon
78 as the rate limit resets, until it succeeds. "pace" works like throw, but tries to wait in
79 between calls so that the limit is generally not hit (How hard it tries to not hit the rate
80 limit can be controlled by ratelimit_pacefactor). The default setting is "wait". Note that
81 even in "wait" and "pace" mode, requests can still fail due to network or other problems! Also
82 note that "pace" and "wait" are NOT thread safe.
83
67 Specify api_base_url if you wish to talk to an instance other than the flagship one. 84 Specify api_base_url if you wish to talk to an instance other than the flagship one.
68 If a file is given as client_id, read client ID and secret from that file 85 If a file is given as client_id, read client ID and secret from that file
69 """ 86 """
@@ -72,6 +89,13 @@ class Mastodon:
72 self.client_secret = client_secret 89 self.client_secret = client_secret
73 self.access_token = access_token 90 self.access_token = access_token
74 self.debug_requests = debug_requests 91 self.debug_requests = debug_requests
92 self.ratelimit_method = ratelimit_method
93
94 self.ratelimit_limit = 150
95 self.ratelimit_reset = time.time()
96 self.ratelimit_remaining = 150
97 self.ratelimit_lastcall = time.time()
98 self.ratelimit_pacefactor = ratelimit_pacefactor
75 99
76 if os.path.isfile(self.client_id): 100 if os.path.isfile(self.client_id):
77 with open(self.client_id, 'r') as secret_file: 101 with open(self.client_id, 'r') as secret_file:
@@ -79,7 +103,7 @@ class Mastodon:
79 self.client_secret = secret_file.readline().rstrip() 103 self.client_secret = secret_file.readline().rstrip()
80 else: 104 else:
81 if self.client_secret == None: 105 if self.client_secret == None:
82 raise ValueError('Specified client id directly, but did not supply secret') 106 raise MastodonIllegalArgumentError('Specified client id directly, but did not supply secret')
83 107
84 if self.access_token != None and os.path.isfile(self.access_token): 108 if self.access_token != None and os.path.isfile(self.access_token):
85 with open(self.access_token, 'r') as token_file: 109 with open(self.access_token, 'r') as token_file:
@@ -87,13 +111,15 @@ class Mastodon:
87 111
88 def log_in(self, username, password, scopes = ['read', 'write', 'follow'], to_file = None): 112 def log_in(self, username, password, scopes = ['read', 'write', 'follow'], to_file = None):
89 """ 113 """
90 Log in and set access_token to what was returned. 114 Log in and sets access_token to what was returned. Note that your
91 Can persist access token to file. 115 username is the e-mail you use to log in into mastodon.
116
117 Can persist access token to file, to be used in the constructor.
92 118
93 Will throw an exception if username / password are wrong, scopes are not 119 Will throw a MastodonIllegalArgumentError if username / password
94 valid or granted scopes differ from requested. 120 are wrong, scopes are not valid or granted scopes differ from requested.
95 121
96 Returns the access_token, as well. 122 Returns the access_token.
97 """ 123 """
98 params = self.__generate_params(locals()) 124 params = self.__generate_params(locals())
99 params['client_id'] = self.client_id 125 params['client_id'] = self.client_id
@@ -102,16 +128,16 @@ class Mastodon:
102 params['scope'] = " ".join(scopes) 128 params['scope'] = " ".join(scopes)
103 129
104 try: 130 try:
105 response = self.__api_request('POST', '/oauth/token', params) 131 response = self.__api_request('POST', '/oauth/token', params, do_ratelimiting = False)
106 self.access_token = response['access_token'] 132 self.access_token = response['access_token']
107 except: 133 except:
108 raise ValueError('Invalid user name, password or scopes.') 134 raise MastodonIllegalArgumentError('Invalid user name, password or scopes.')
109 135
110 requested_scopes = " ".join(sorted(scopes)) 136 requested_scopes = " ".join(sorted(scopes))
111 received_scopes = " ".join(sorted(response["scope"].split(" "))) 137 received_scopes = " ".join(sorted(response["scope"].split(" ")))
112 138
113 if requested_scopes != received_scopes: 139 if requested_scopes != received_scopes:
114 raise ValueError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".') 140 raise MastodonAPIError('Granted scopes "' + received_scopes + '" differ from requested scopes "' + requested_scopes + '".')
115 141
116 if to_file != None: 142 if to_file != None:
117 with open(to_file, 'w') as token_file: 143 with open(to_file, 'w') as token_file:
@@ -381,19 +407,18 @@ class Mastodon:
381 type has to be specified manually, otherwise, it is 407 type has to be specified manually, otherwise, it is
382 determined from the file name. 408 determined from the file name.
383 409
384 Throws a ValueError if the mime type of the passed data or file can 410 Throws a MastodonIllegalArgumentError if the mime type of the
385 not be determined properly. 411 passed data or file can not be determined properly.
386 412
387 Returns a media dict. This contains the id that can be used in 413 Returns a media dict. This contains the id that can be used in
388 status_post to attach the media file to a toot. 414 status_post to attach the media file to a toot.
389 """ 415 """
390
391 if os.path.isfile(media_file) and mime_type == None: 416 if os.path.isfile(media_file) and mime_type == None:
392 mime_type = mimetypes.guess_type(media_file)[0] 417 mime_type = mimetypes.guess_type(media_file)[0]
393 media_file = open(media_file, 'rb') 418 media_file = open(media_file, 'rb')
394 419
395 if mime_type == None: 420 if mime_type == None:
396 raise ValueError('Could not determine mime type or data passed directly without mime type.') 421 raise MastodonIllegalArgumentError('Could not determine mime type or data passed directly without mime type.')
397 422
398 random_suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) 423 random_suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
399 file_name = "mastodonpyupload_" + str(time.time()) + "_" + str(random_suffix) + mimetypes.guess_extension(mime_type) 424 file_name = "mastodonpyupload_" + str(time.time()) + "_" + str(random_suffix) + mimetypes.guess_extension(mime_type)
@@ -404,14 +429,50 @@ class Mastodon:
404 ### 429 ###
405 # Internal helpers, dragons probably 430 # Internal helpers, dragons probably
406 ### 431 ###
407 def __api_request(self, method, endpoint, params = {}, files = {}): 432 def __datetime_to_epoch(self, date_time):
433 """
434 Converts a python datetime to unix epoch, accounting for
435 time zones and such.
436
437 Assumes UTC if timezone is not given.
438 """
439 date_time_utc = None
440 if date_time.tzinfo == None:
441 date_time_utc = date_time.replace(tzinfo = pytz.utc)
442 else:
443 date_time_utc = date_time.astimezone(pytz.utc)
444
445 epoch_utc = datetime.datetime.utcfromtimestamp(0).replace(tzinfo = pytz.utc)
446
447 return (date_time_utc - epoch_utc).total_seconds()
448
449 def __api_request(self, method, endpoint, params = {}, files = {}, do_ratelimiting = True):
408 """ 450 """
409 Internal API request helper. 451 Internal API request helper.
410 """ 452 """
411 response = None 453 response = None
412 headers = None 454 headers = None
413 455
414 456 # "pace" mode ratelimiting: Assume constant rate of requests, sleep a little less long than it
457 # would take to not hit the rate limit at that request rate.
458 if do_ratelimiting and self.ratelimit_method == "pace":
459 if self.ratelimit_remaining == 0:
460 to_next = self.ratelimit_reset - time.time()
461 if to_next > 0:
462 # As a precaution, never sleep longer than 5 minutes
463 to_next = min(to_next, 5 * 60)
464 time.sleep(to_next)
465 else:
466 time_waited = time.time() - self.ratelimit_lastcall
467 time_wait = float(self.ratelimit_reset - time.time()) / float(self.ratelimit_remaining)
468 remaining_wait = time_wait - time_waited
469
470 if remaining_wait > 0:
471 to_next = remaining_wait / self.ratelimit_pacefactor
472 to_next = min(to_next, 5 * 60)
473 time.sleep(to_next)
474
475 # Generate request headers
415 if self.access_token != None: 476 if self.access_token != None:
416 headers = {'Authorization': 'Bearer ' + self.access_token} 477 headers = {'Authorization': 'Bearer ' + self.access_token}
417 478
@@ -421,26 +482,74 @@ class Mastodon:
421 print('Headers: ' + str(headers)) 482 print('Headers: ' + str(headers))
422 print('Files: ' + str(files)) 483 print('Files: ' + str(files))
423 484
424 if method == 'GET': 485 # Make request
425 response = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files) 486 request_complete = False
426 487 while not request_complete:
427 if method == 'POST': 488 request_complete = True
428 response = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
429 489
430 if method == 'DELETE': 490 response_object = None
431 response = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files) 491 try:
432 492 if method == 'GET':
433 if response.status_code == 404: 493 response_object = requests.get(self.api_base_url + endpoint, data = params, headers = headers, files = files)
434 raise IOError('Endpoint not found.') 494
435 495 if method == 'POST':
436 if response.status_code == 500: 496 response_object = requests.post(self.api_base_url + endpoint, data = params, headers = headers, files = files)
437 raise IOError('General API problem.') 497
438 498 if method == 'DELETE':
439 try: 499 response_object = requests.delete(self.api_base_url + endpoint, data = params, headers = headers, files = files)
440 response = response.json() 500 except:
441 except: 501 raise MastodonNetworkError("Could not complete request.")
442 raise ValueError("Could not parse response as JSON, respose code was " + str(response.status_code)) 502
443 503 if response_object == None:
504 raise MastodonIllegalArgumentError("Illegal request.")
505
506 # Handle response
507 if self.debug_requests == True:
508 print('Mastodon: Response received with code ' + str(response_object.status_code) + '.')
509 print('Respose headers: ' + str(response_object.headers))
510 print('Response text content: ' + str(response_object.text))
511
512 if response_object.status_code == 404:
513 raise MastodonAPIError('Endpoint not found.')
514
515 if response_object.status_code == 500:
516 raise MastodonAPIError('General API problem.')
517
518 try:
519 response = response_object.json()
520 except:
521 raise MastodonAPIError("Could not parse response as JSON, respose code was " + str(response_object.status_code))
522
523 # Handle rate limiting
524 try:
525 if 'X-RateLimit-Remaining' in response_object.headers and do_ratelimiting:
526 self.ratelimit_remaining = int(response_object.headers['X-RateLimit-Remaining'])
527 self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit'])
528
529 ratelimit_reset_datetime = dateutil.parser.parse(response_object.headers['X-RateLimit-Reset'])
530 self.ratelimit_reset = self.__datetime_to_epoch(ratelimit_reset_datetime)
531
532 # Adjust server time to local clock
533 server_time_datetime = dateutil.parser.parse(response_object.headers['Date'])
534 server_time = self.__datetime_to_epoch(server_time_datetime)
535 server_time_diff = time.time() - server_time
536 self.ratelimit_reset += server_time_diff
537 self.ratelimit_lastcall = time.time()
538
539 if "error" in response and response["error"] == "Throttled":
540 if self.ratelimit_method == "throw":
541 raise MastodonRatelimitError("Hit rate limit.")
542
543 if self.ratelimit_method == "wait" or self.ratelimit_method == "pace":
544 to_next = self.ratelimit_reset - time.time()
545 if to_next > 0:
546 # As a precaution, never sleep longer than 5 minutes
547 to_next = min(to_next, 5 * 60)
548 time.sleep(to_next)
549 request_complete = False
550 except:
551 raise MastodonRatelimitError("Rate limit time calculations failed.")
552
444 return response 553 return response
445 554
446 def __generate_params(self, params, exclude = []): 555 def __generate_params(self, params, exclude = []):
@@ -462,3 +571,22 @@ class Mastodon:
462 del params[key] 571 del params[key]
463 572
464 return params 573 return params
574
575##
576# Exceptions
577##
578class MastodonIllegalArgumentError(ValueError):
579 pass
580
581class MastodonFileNotFoundError(IOError):
582 pass
583
584class MastodonNetworkError(IOError):
585 pass
586
587class MastodonAPIError(Exception):
588 pass
589
590class MastodonRatelimitError(Exception):
591 pass
592
Powered by cgit v1.2.3 (git 2.41.0)