2 files changed, 162 insertions, 48 deletions

diff --git a/mastodon/Mastodon.py b/mastodon/Mastodon.py
index bdf7f1b..296921e 100644
--- a/mastodon/Mastodon.py
+++ b/mastodon/Mastodon.py
@@ -120,10 +120,27 @@ class AttribAccessDict(dict):
             raise AttributeError("Attribute-style access is read only")
         super(AttribAccessDict, self).__setattr__(attr, val)
 
+
 ###
-# The actual Mastodon class
+# List helper class.
+# Defined at top level so it can be pickled.
 ###
+class AttribAccessList(list):
+    def __getattr__(self, attr):
+        if attr in self:
+            return self[attr]
+        else:
+            raise AttributeError("Attribute not found: " + str(attr))
+
+    def __setattr__(self, attr, val):
+        if attr in self:
+            raise AttributeError("Attribute-style access is read only")
+        super(AttribAccessList, self).__setattr__(attr, val)
+
 
+###
+# The actual Mastodon class
+###
 class Mastodon:
     """
     Thorough and easy to use Mastodon
@@ -276,6 +293,7 @@ class Mastodon:
                 secret_file.write(response['client_id'] + "\n")
                 secret_file.write(response['client_secret'] + "\n")
                 secret_file.write(api_base_url + "\n")
+                secret_file.write(client_name + "\n")
                 
         return (response['client_id'], response['client_secret'])
 
@@ -286,7 +304,7 @@ class Mastodon:
                  api_base_url=None, debug_requests=False,
                  ratelimit_method="wait", ratelimit_pacefactor=1.1,
                  request_timeout=__DEFAULT_TIMEOUT, mastodon_version=None,
-                 version_check_mode = "created", session=None, feature_set="mainline"):
+                 version_check_mode = "created", session=None, feature_set="mainline", user_agent=None):
         """
         Create a new API wrapper instance based on the given `client_secret` and `client_id`. If you
         give a `client_id` and it is not a file, you must also give a secret. If you specify an
@@ -331,6 +349,11 @@ class Mastodon:
         `feature_set` can be used to enable behaviour specific to non-mainline Mastodon API implementations.
         Details are documented in the functions that provide such functionality. Currently supported feature
         sets are `mainline`, `fedibird` and `pleroma`.
+
+        For some mastodon-instances a `User-Agent` header is needed. This can be set by parameter `user_agent`. From now
+        `create_app()` stores the application name into the client secret file. If `client_id` points to this file,
+        the app name will be used as `User-Agent` header as default. It's possible to modify old secret files and append
+        a client app name to use it as a `User-Agent` name.
         """
         self.api_base_url = None
         if not api_base_url is None:
@@ -362,6 +385,9 @@ class Mastodon:
         self.feature_set = feature_set
         if not self.feature_set in ["mainline", "fedibird", "pleroma"]:
             raise MastodonIllegalArgumentError('Requested invalid feature set')
+
+        # General defined user-agent
+        self.user_agent = user_agent
         
         # Token loading
         if self.client_id is not None:
@@ -376,6 +402,11 @@ class Mastodon:
                         if not (self.api_base_url is None or try_base_url == self.api_base_url):
                             raise MastodonIllegalArgumentError('Mismatch in base URLs between files and/or specified')
                         self.api_base_url = try_base_url
+
+                    # With new registrations we support the 4th line to store a client_name and use it as user-agent
+                    client_name = secret_file.readline()
+                    if client_name and self.user_agent is None:
+                        self.user_agent = client_name.rstrip()
             else:
                 if self.client_secret is None:
                     raise MastodonIllegalArgumentError('Specified client id directly, but did not supply secret')
@@ -390,20 +421,20 @@ class Mastodon:
                     if not (self.api_base_url is None or try_base_url == self.api_base_url):
                          raise MastodonIllegalArgumentError('Mismatch in base URLs between files and/or specified')
                     self.api_base_url = try_base_url
+
+        if not version_check_mode in ["created", "changed", "none"]:
+            raise MastodonIllegalArgumentError("Invalid version check method.")
+        self.version_check_mode = version_check_mode
         
         # Versioning
-        if mastodon_version == None:
+        if mastodon_version == None and self.version_check_mode != 'none':
             self.retrieve_mastodon_version()
-        else:
+        elif self.version_check_mode != 'none':
             try:
                 self.mastodon_major, self.mastodon_minor, self.mastodon_patch = parse_version_string(mastodon_version)
             except:
                 raise MastodonVersionError("Bad version specified")
         
-        if not version_check_mode in ["created", "changed", "none"]:
-            raise MastodonIllegalArgumentError("Invalid version check method.")
-        self.version_check_mode = version_check_mode
-        
         # Ratelimiting parameter check
         if ratelimit_method not in ["throw", "wait", "pace"]:
             raise MastodonIllegalArgumentError("Invalid ratelimit method.")
@@ -965,10 +996,13 @@ class Mastodon:
     # Reading data: Notifications
     ###
     @api_version("1.0.0", "2.9.0", __DICT_VERSION_NOTIFICATION)
-    def notifications(self, id=None, account_id=None, max_id=None, min_id=None, since_id=None, limit=None, mentions_only=None):
+    def notifications(self, id=None, account_id=None, max_id=None, min_id=None, since_id=None, limit=None, exclude_types=None):
         """
         Fetch notifications (mentions, favourites, reblogs, follows) for the logged-in
         user. Pass `account_id` to get only notifications originating from the given account.
+
+        Parameter `exclude_types` is an array of the following `follow`, `favourite`, `reblog`,
+        `mention`, `poll`, `follow_request`
         
         Can be passed an `id` to fetch a single notification.
 
@@ -1028,8 +1062,8 @@ class Mastodon:
         """
         return self.account_verify_credentials()
     
-    @api_version("1.0.0", "2.7.0", __DICT_VERSION_STATUS)
-    def account_statuses(self, id, only_media=False, pinned=False, exclude_replies=False, max_id=None, min_id=None, since_id=None, limit=None):
+    @api_version("1.0.0", "2.8.0", __DICT_VERSION_STATUS)
+    def account_statuses(self, id, only_media=False, pinned=False, exclude_replies=False, exclude_reblogs=False, tagged=None, max_id=None, min_id=None, since_id=None, limit=None):
         """
         Fetch statuses by user `id`. Same options as `timeline()`_ are permitted.
         Returned toots are from the perspective of the logged-in user, i.e.
@@ -1040,6 +1074,8 @@ class Mastodon:
         If `pinned` is set, return only statuses that have been pinned. Note that 
         as of Mastodon 2.1.0, this only works properly for instance-local users.
         If `exclude_replies` is set, filter out all statuses that are replies.
+        If `exclude_reblogs` is set, filter out all statuses that are reblogs.
+        If `tagged` is set, return only statuses that are tagged with `tagged`. Only a single tag without a '#' is valid.
 
         Does not require authentication for Mastodon versions after 2.7.0 (returns
         publicly visible statuses in that case), for publicly visible accounts.
@@ -1063,7 +1099,9 @@ class Mastodon:
             del params["only_media"]
         if exclude_replies == False:
             del params["exclude_replies"]
-        
+        if exclude_reblogs == False:
+            del params["exclude_reblogs"]
+
         url = '/api/v1/accounts/{0}/statuses'.format(str(id))
         return self.__api_request('GET', url, params)
 
@@ -1633,13 +1671,23 @@ class Mastodon:
     # Reading data: Bookmarks
     ###
     @api_version("3.1.0", "3.1.0", __DICT_VERSION_STATUS)
-    def bookmarks(self):
+    def bookmarks(self, max_id=None, min_id=None, since_id=None, limit=None):
         """
         Get a list of statuses bookmarked by the logged-in user.
         
         Returns a list of `toot dicts`_.
         """
-        return self.__api_request('GET', '/api/v1/bookmarks')
+        if max_id != None:
+            max_id = self.__unpack_id(max_id)
+
+        if min_id != None:
+            min_id = self.__unpack_id(min_id)
+
+        if since_id != None:
+            since_id = self.__unpack_id(since_id)
+
+        params = self.__generate_params(locals())
+        return self.__api_request('GET', '/api/v1/bookmarks', params)
     
     ###
     # Writing data: Statuses
@@ -2424,7 +2472,7 @@ class Mastodon:
         if not status_ids is None:
             if not isinstance(status_ids, list):
                 status_ids = [status_ids]
-        status_ids = list(map(lambda x: self.__unpack_id(x), status_ids))
+            status_ids = list(map(lambda x: self.__unpack_id(x), status_ids))
         
         params_initial = locals()        
         if forward == False:
@@ -3041,8 +3089,8 @@ class Mastodon:
         Returns the next page or None if no further data is available.
         """
         if isinstance(previous_page, list) and len(previous_page) != 0:
-            if hasattr(previous_page[-1], '_pagination_next'):
-                params = copy.deepcopy(previous_page[-1]._pagination_next)
+            if hasattr(previous_page, '_pagination_next'):
+                params = copy.deepcopy(previous_page._pagination_next)
             else:
                 return None
         else:
@@ -3065,8 +3113,8 @@ class Mastodon:
         Returns the previous page or None if no further data is available.
         """
         if isinstance(next_page, list) and len(next_page) != 0:
-            if hasattr(next_page[0], '_pagination_prev'):
-                params = copy.deepcopy(next_page[0]._pagination_prev)
+            if hasattr(next_page, '_pagination_prev'):
+                params = copy.deepcopy(next_page._pagination_prev)
             else:
                 return None
         else:
@@ -3231,7 +3279,7 @@ class Mastodon:
             if (key in json_object and isinstance(json_object[key], six.text_type)):
                 if json_object[key].lower() == 'true':
                     json_object[key] = True
-                if json_object[key].lower() == 'False':
+                if json_object[key].lower() == 'false':
                     json_object[key] = False
         return json_object
     
@@ -3305,6 +3353,10 @@ class Mastodon:
         if not access_token_override is None:
             headers['Authorization'] = 'Bearer ' + access_token_override
 
+        # Add user-agent
+        if self.user_agent:
+            headers['User-Agent'] = self.user_agent
+
         # Determine base URL
         base_url = self.api_base_url
         if not base_url_override is None:
@@ -3356,8 +3408,11 @@ class Mastodon:
                 self.ratelimit_limit = int(response_object.headers['X-RateLimit-Limit'])
 
                 try:
-                    ratelimit_reset_datetime = dateutil.parser.parse(response_object.headers['X-RateLimit-Reset'])
-                    self.ratelimit_reset = self.__datetime_to_epoch(ratelimit_reset_datetime)
+                    if str(int(response_object.headers['X-RateLimit-Reset'])) == response_object.headers['X-RateLimit-Reset']:
+                        self.ratelimit_reset = int(response_object.headers['X-RateLimit-Reset'])
+                    else:
+                        ratelimit_reset_datetime = dateutil.parser.parse(response_object.headers['X-RateLimit-Reset'])
+                        self.ratelimit_reset = self.__datetime_to_epoch(ratelimit_reset_datetime)
 
                     # Adjust server time to local clock
                     if 'Date' in response_object.headers:
@@ -3444,6 +3499,7 @@ class Mastodon:
             if isinstance(response, list) and \
                     'Link' in response_object.headers and \
                     response_object.headers['Link'] != "":
+                response = AttribAccessList(response)
                 tmp_urls = requests.utils.parse_header_links(
                     response_object.headers['Link'].rstrip('>').replace('>,<', ',<'))
                 for url in tmp_urls:
@@ -3468,7 +3524,12 @@ class Mastodon:
                                 del next_params['since_id']
                             if "min_id" in next_params:
                                 del next_params['min_id']
-                            response[-1]._pagination_next = next_params
+                            response._pagination_next = next_params
+
+                            # Maybe other API users rely on the pagination info in the last item
+                            # Will be removed in future
+                            if isinstance(response[-1], AttribAccessDict):
+                                response[-1]._pagination_next = next_params
 
                     if url['rel'] == 'prev':
                         # Be paranoid and extract since_id or min_id specifically
@@ -3487,8 +3548,13 @@ class Mastodon:
                                 prev_params['since_id'] = since_id
                             if "max_id" in prev_params:
                                 del prev_params['max_id']
-                            response[0]._pagination_prev = prev_params
-                            
+                            response._pagination_prev = prev_params
+
+                            # Maybe other API users rely on the pagination info in the first item
+                            # Will be removed in future
+                            if isinstance(response[0], AttribAccessDict):
+                                response[0]._pagination_prev = prev_params
+
                         # New and fantastico (post-2.6.0): min_id pagination
                         matchgroups = re.search(r"[?&]min_id=([^&]+)", prev_url)
                         if matchgroups:
@@ -3502,7 +3568,12 @@ class Mastodon:
                                 prev_params['min_id'] = min_id
                             if "max_id" in prev_params:
                                 del prev_params['max_id']
-                            response[0]._pagination_prev = prev_params
+                            response._pagination_prev = prev_params
+
+                            # Maybe other API users rely on the pagination info in the first item
+                            # Will be removed in future
+                            if isinstance(response[0], AttribAccessDict):
+                                response[0]._pagination_prev = prev_params
 
         return response
 
@@ -3547,6 +3618,8 @@ class Mastodon:
         # Connect function (called and then potentially passed to async handler)
         def connect_func():
             headers = {"Authorization": "Bearer " + self.access_token} if self.access_token else {}
+            if self.user_agent:
+                headers['User-Agent'] = self.user_agent
             connection = self.session.get(url + endpoint, headers = headers, data = params, stream = True,
                                   timeout=(self.request_timeout, timeout))
 
@@ -3568,7 +3641,8 @@ class Mastodon:
                 
             def close(self):
                 self.closed = True
-                self.connection.close()
+                if not self.connection is None:
+                    self.connection.close()
 
             def is_alive(self):
                 return self._thread.is_alive()
@@ -3579,6 +3653,14 @@ class Mastodon:
                 else:
                     return True
 
+            def _sleep_attentive(self):
+                if self._thread != threading.current_thread():
+                    raise RuntimeError ("Illegal call from outside the stream_handle thread")
+                time_remaining = self.reconnect_async_wait_sec
+                while time_remaining>0 and not self.closed:
+                    time.sleep(0.5)
+                    time_remaining -= 0.5
+
             def _threadproc(self):
                 self._thread = threading.current_thread()
                 
@@ -3600,16 +3682,26 @@ class Mastodon:
                         self.reconnecting = True
                         connect_success = False
                         while not connect_success:
-                            connect_success = True
+                            if self.closed:
+                                # Someone from outside stopped the streaming
+                                self.running = False
+                                break
                             try:
-                                self.connection = self.connect_func()
-                                if self.connection.status_code != 200:
-                                    time.sleep(self.reconnect_async_wait_sec)
-                                    connect_success = False
-                                    exception = MastodonNetworkError("Could not connect to server.")
+                                the_connection = self.connect_func()
+                                if the_connection.status_code != 200:
+                                    exception = MastodonNetworkError(f"Could not connect to server. "
+                                                                     f"HTTP status: {the_connection.status_code}")
                                     listener.on_abort(exception)
+                                    self._sleep_attentive()
+                                if self.closed:
+                                    # Here we have maybe a rare race condition. Exactly on connect, someone
+                                    # stopped the streaming before. We close the previous established connection:
+                                    the_connection.close()
+                                else:
+                                    self.connection = the_connection
+                                    connect_success = True
                             except:
-                                time.sleep(self.reconnect_async_wait_sec)
+                                self._sleep_attentive()
                                 connect_success = False
                         self.reconnecting = False
                     else:
diff --git a/mastodon/streaming.py b/mastodon/streaming.py
index 214ed1c..ceb61ea 100644
--- a/mastodon/streaming.py
+++ b/mastodon/streaming.py
@@ -45,6 +45,16 @@ class StreamListener(object):
         contains the resulting conversation dict."""
         pass
 
+    def on_unknown_event(self, name, unknown_event = None):
+        """An unknown mastodon API event has been received. The name contains the event-name and unknown_event
+        contains the content of the unknown event.
+
+        This function must be implemented, if unknown events should be handled without an error.
+        """
+        exception = MastodonMalformedEventError('Bad event type', name)
+        self.on_abort(exception)
+        raise exception
+
     def handle_heartbeat(self):
         """The server has sent us a keep-alive message. This callback may be
         useful to carry out periodic housekeeping tasks, or just to confirm
@@ -56,6 +66,11 @@ class StreamListener(object):
         Handles a stream of events from the Mastodon server. When each event
         is received, the corresponding .on_[name]() method is called.
 
+        When the Mastodon API changes, the on_unknown_event(name, content)
+        function is called.
+        The default behavior is to throw an error. Define a callback handler
+        to intercept unknown events if needed (and avoid errors)
+
         response; a requests response object with the open stream for reading.
         """
         event = {}
@@ -137,33 +152,32 @@ class StreamListener(object):
                 exception,
                 err
             )
-           
-        handler_name = 'on_' + name
-        try:
-            handler = getattr(self, handler_name)
-        except AttributeError as err:
-            exception = MastodonMalformedEventError('Bad event type', name)
-            self.on_abort(exception)
-            six.raise_from(
-               exception,
-               err
-            )
-        else:
+        # New mastodon API also supports event names with dots:
+        handler_name = 'on_' + name.replace('.', '_')
+        # A generic way to handle unknown events to make legacy code more stable for future changes
+        handler = getattr(self, handler_name, self.on_unknown_event)
+        if handler != self.on_unknown_event:
             handler(payload)
+        else:
+            handler(name, payload)
+
 
 class CallbackStreamListener(StreamListener):
     """
     Simple callback stream handler class.
     Can optionally additionally send local update events to a separate handler.
+    Define an unknown_event_handler for new Mastodon API events. If not, the
+    listener will raise an error on new, not handled, events from the API.
     """
-    def __init__(self, update_handler = None, local_update_handler = None, delete_handler = None, notification_handler = None, conversation_handler = None):
+    def __init__(self, update_handler = None, local_update_handler = None, delete_handler = None, notification_handler = None, conversation_handler = None, unknown_event_handler = None):
         super(CallbackStreamListener, self).__init__()
         self.update_handler = update_handler
         self.local_update_handler = local_update_handler
         self.delete_handler = delete_handler
         self.notification_handler = notification_handler
         self.conversation_handler = conversation_handler
-        
+        self.unknown_event_handler = unknown_event_handler
+
     def on_update(self, status):
         if self.update_handler != None:
             self.update_handler(status)
@@ -188,3 +202,11 @@ class CallbackStreamListener(StreamListener):
     def on_conversation(self, conversation):
         if self.conversation_handler != None:
             self.conversation_handler(conversation)            
+
+    def on_unknown_event(self, name, unknown_event = None):
+        if self.unknown_event_handler != None:
+            self.unknown_event_handler(name, unknown_event)
+        else:
+            exception = MastodonMalformedEventError('Bad event type', name)
+            self.on_abort(exception)
+            raise exception