Fix #4

README.md

@@ -25,4 +25,6 @@ Within Python Virtual Environment:
 29.9.2021 **New Feature** Added support to media files! mastotuit now gets all media files from Mastodon's post (if any) and publish them to Twitter together with your status update.  
 7.10.2021 **New Feature** Added thread support! If you create a thread in Mastodon mastotuit will create the same thread on Twitter.  
 13.10.2021 Upgraded Tweepy library to v4.1.0  
-13.10.2021 **New Feature** Added video upload support! If video properties are according Twitter rules it will be uploaded.
+13.10.2021 **New Feature** Added video upload support! If video properties are according Twitter rules it will be uploaded.  
+14.2.2022 Upgraded Tweepy library to v4.5.0  
+14.2.2022 **New Feature** Polls support! Now your Mastodon's polls are replicated to Twitter and they can vote them!

db-setup.py

@@ -4,9 +4,20 @@ import sys
 import psycopg2
 from psycopg2 import sql
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+import pdb
+
+def get_dbconfig():
+
+    # Load configuration from config file
+    config_filepath = "config/db_config.txt"
+    feeds_db = get_parameter("feeds_db", config_filepath)
+    feeds_db_user = get_parameter("feeds_db_user", config_filepath)
+    feeds_url = get_parameter("feeds_url", config_filepath)
+
+    return (config_filepath, feeds_db, feeds_db_user, feeds_url)
 
-# Returns the parameter from the specified file
 def get_parameter( parameter, file_path ):
+
     # Check if secrets file exists
     if not os.path.isfile(file_path):
         print("File %s not found, asking."%file_path)
@@ -23,7 +34,67 @@ def get_parameter( parameter, file_path ):
     print(file_path + "  Missing parameter %s "%parameter)
     sys.exit(0)
 
+def create_db():
+
+    create_error = ''
+
+    conn = None
+
+    try:
+
+      conn = psycopg2.connect(dbname='postgres',
+          user=feeds_db_user, host='',
+          password='')
+
+      conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+
+      cur = conn.cursor()
+
+      print("Creating database " + feeds_db + ". Please wait...")
+
+      cur.execute(sql.SQL("CREATE DATABASE {}").format(
+              sql.Identifier(feeds_db))
+          )
+      print("Database " + feeds_db + " created!")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        create_error = error.pgcode
+
+        sys.stdout.write(f'\n{str(error)}\n')
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
+    return create_error
+
+def check_db_conn():
+
+    try:
+
+        conn = None
+
+        conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+        os.remove(config_filepath)
+
+        sys.exit('Exiting. Run db-setup again with right parameters')
+
+    if conn is not None:
+
+        print("\n")
+        print("mastotuit parameters saved to db-config.txt!")
+        print("\n")
+
 def write_parameter( parameter, file_path ):
+
     if not os.path.exists('config'):
         os.makedirs('config')
     print("Setting up mastotuit parameters...")
@@ -64,84 +135,39 @@ def create_table(db, db_user, table, sql):
       conn.close()
 
 #############################################################################################
+# main
 
-# Load configuration from config file
-config_filepath = "config/db_config.txt"
-feeds_db = get_parameter("feeds_db", config_filepath)
-feeds_db_user = get_parameter("feeds_db_user", config_filepath)
-feeds_url = get_parameter("feeds_url", config_filepath)
+if __name__ == '__main__':
 
-############################################################
-# create database
-############################################################
+    config_filepath, feeds_db, feeds_db_user, feeds_url = get_dbconfig()
 
-conn = None
+    create_error = create_db()
 
-try:
+    if create_error == '':
 
-  conn = psycopg2.connect(dbname='postgres',
-      user=feeds_db_user, host='',
-      password='')
+        check_db_conn()
 
-  conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
+    else:
 
-  cur = conn.cursor()
+        if create_error == '42P04':
 
-  print("Creating database " + feeds_db + ". Please wait...")
+            sys.exit()
 
-  cur.execute(sql.SQL("CREATE DATABASE {}").format(
-          sql.Identifier(feeds_db))
-      )
-  print("Database " + feeds_db + " created!")
+        else:
 
-except (Exception, psycopg2.DatabaseError) as error:
+            os.remove(config_filepath)
 
-  print(error)
+            sys.exit()
 
-finally:
+    ############################################################
+    # Create needed tables
+    ############################################################
 
-  if conn is not None:
+    db = feeds_db
+    db_user = feeds_db_user
+    table = "id"
+    sql = f'create table {table} (toot_id bigint PRIMARY KEY, tweet_id bigint)'
 
-    conn.close()
+    create_table(db, db_user, table, sql)
 
-#############################################################################################
-
-try:
-
-  conn = None
-  conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-except (Exception, psycopg2.DatabaseError) as error:
-
-  print(error)
-  # Load configuration from config file
-  os.remove("db_config.txt")
-  print("Exiting. Run db-setup again with right parameters")
-  sys.exit(0)
-
-if conn is not None:
-
-  print("\n")
-  print("mastotuit parameters saved to db-config.txt!")
-  print("\n")
-
-############################################################
-# Create needed tables
-############################################################
-
-print("Creating table...")
-
-########################################
-
-db = feeds_db
-db_user = feeds_db_user
-
-table = "id"
-sql = "create table "+table+" (toot_id bigint PRIMARY KEY, tweet_id bigint)"
-create_table(db, db_user, table, sql)
-
-#####################################
-
-print("Done!")
-print("Now you can run setup.py!")
-print("\n")
+    print(f'Done!\nNow you can run setup.py\n')

mastotuit.py

@@ -12,44 +12,99 @@ from tweepy import TweepyException
 import logging
 import filetype
 import ffmpeg
-import pdb
 
 logger = logging.getLogger()
 
-def get_toot(title):
+def get_toot_text(title):
 
-    soup = BeautifulSoup(title, 'html.parser')
+    soup = BeautifulSoup(title, features='html.parser')
 
-    toot_text = soup.get_text()
+    delimiter = '###' # unambiguous string
 
-    sub_str = 'http'
-    find_link = toot_text.find(sub_str)
-    if find_link != -1:
+    for line_break in soup.findAll('br'):   # loop through line break tags
 
-        tuit_text = toot_text[:toot_text.index(sub_str)]
+        line_break.replaceWith(delimiter)   # replace br tags with delimiter
 
-    else:
+    tuit_text_str = soup.get_text().split(delimiter)  # get list of strings
 
-        tuit_text = toot_text
+    tuit_text = ''
 
-    links_lst = ''
-    for links in soup.find_all('a'):
-        find_tag = links.get('href').find('/tags/')
-        if find_tag == -1:
-            links_lst += links.get('href')
+    for line in tuit_text_str:
 
-    if len(links_lst) > 0:
-
-        last_text = toot_text[len(tuit_text) + len(links_lst):]
-
-    else:
-
-        last_text = ''
-
-    tuit_text = f'{tuit_text} {links_lst} {last_text}'
+        tuit_text += f'{line}\n'
 
     return tuit_text
 
+def get_poll(tuit_text):
+
+    poll_options = 0
+
+    options_lst = []
+
+    is_poll = False if "input disable" not in entry.summary else True
+
+    if is_poll:
+
+        poll_substring = """<input disabled="disabled" type="radio" />"""
+
+        poll_options = title.count(poll_substring)
+
+        remaining_str = title
+
+        i = poll_options
+        while (i > 0):
+
+            last_option_index = remaining_str.rfind(poll_substring)
+
+            if i == poll_options:
+
+                option_str = remaining_str[last_option_index+42:].strip().replace('</p>','')
+
+            else:
+
+                option_str = remaining_str[last_option_index+42:].strip().replace('<br />','')
+
+            options_lst.append(option_str)
+            remaining_str = remaining_str[:last_option_index]
+            i-=1
+
+        options_lst_copy = options_lst.copy()
+        options_lst_copy.reverse()
+        options_lst = options_lst_copy.copy()
+
+        first_option_index = tuit_text.rfind(options_lst[0])
+
+        tuit_text = tuit_text[:first_option_index-1]
+
+    return (tuit_text, poll_options, options_lst, is_poll)
+
+def get_toot(title):
+
+    tuit_text = get_toot_text(title)
+
+    tuit_text, poll_options, options_lst, is_poll = get_poll(tuit_text)
+
+    return (tuit_text, poll_options, options_lst, is_poll)
+
+def compose_poll(tuit_text, poll_options, options_lst, toot_id):
+
+    try:
+
+        tweet = apiv2.create_tweet(
+            poll_duration_minutes=4320,
+            poll_options = options_lst,
+            text=tuit_text
+        )
+
+        write_db(toot_id, tweet.data['id'])
+
+    except TweepyException as err:
+
+        print('Error while trying to publish poll.\n')
+        sys.exit(err)
+
+    return tweet
+
 def compose_tweet(tuit_text, with_images, is_reply):
 
     images_id_lst = []
@@ -82,7 +137,7 @@ def compose_tweet(tuit_text, with_images, is_reply):
 
             try:
 
-                media_upload = api.media_upload(
+                media_upload = apiv1.media_upload(
                     f'images/{image}',
                     media_category='tweet_video' if is_video else 'tweet_image'
                 )
@@ -105,30 +160,45 @@ def compose_tweet(tuit_text, with_images, is_reply):
     # Compose tuit
     tuit_text2 = ''
 
+    three_parts = False
+
     if len(tuit_text) > 280:
 
-        tuit_max_length = 250 if with_images else 275
+        tuit_max_length = 250 if with_images else 273
 
-        tuit_text1 = '{0} (1/2)'.format(
-            tuit_text[:tuit_max_length].rsplit(' ', 1)[0]
-        )
-        tuit_text2 = '{0} (2/2)'.format(
-            tuit_text[len(tuit_text1) - 6:]
-        )
+        tuit_text1 = '{0}...'.format(tuit_text[:tuit_max_length].rsplit(' ', 1)[0])
+
+        tuit_text2 = '{0}'.format(tuit_text[len(tuit_text1) - 2:])
+
+        if len(tuit_text2) > 250:
+
+            three_parts = True 
+
+            tuit_text2 = '{0}'.format(tuit_text[len(tuit_text1) - 2:].rsplit('#', 1)[0])
+
+            tuit_text3 = '#{0}'.format(tuit_text[len(tuit_text1) - 2:].rsplit('#', 1)[1].rsplit(' ', 2)[0])
 
         try:
 
-            first_tweet = api.update_status(
+            first_tweet = apiv1.update_status(
                 status=tuit_text1,
-                in_reply_to_status_id=tweet_id if is_reply else ''
-            )
-
-            tweet = api.update_status(
-                status=tuit_text2,
-                in_reply_to_status_id=first_tweet.id,
+                in_reply_to_status_id=tweet_id if is_reply else '',
                 media_ids=images_id_lst
             )
 
+            tweet = apiv1.update_status(
+                status=tuit_text2,
+                in_reply_to_status_id=first_tweet.id
+                #media_ids=images_id_lst
+            )
+
+            if three_parts:
+
+                tweet = apiv1.update_status(
+                    status=tuit_text3,
+                    in_reply_to_status_id=tweet.id
+                )
+
         except TweepyException as err:
 
             print('Error while trying to publish split tweet.\n')
@@ -138,7 +208,7 @@ def compose_tweet(tuit_text, with_images, is_reply):
 
         try:
 
-            tweet = api.update_status(
+            tweet = apiv1.update_status(
                 status=tuit_text,
                 in_reply_to_status_id=tweet_id if is_reply else '',
                 media_ids=images_id_lst
@@ -185,6 +255,34 @@ def get_tweet_id(toot_id):
 
             conn.close()
 
+def write_db(toot_id, tweet_id):
+
+    sql_insert_ids = 'INSERT INTO id(toot_id, tweet_id) VALUES (%s,%s)'
+
+    conn = None
+
+    try:
+
+        conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
+
+        cur = conn.cursor()
+
+        cur.execute(sql_insert_ids, (toot_id, tweet_id))
+
+        conn.commit()
+
+        cur.close()
+
+    except (Exception, psycopg2.DatabaseError) as error:
+
+        print(error)
+
+    finally:
+
+        if conn is not None:
+
+            conn.close()
+
 def write_image(image_url):
 
     if not os.path.exists('images'):
@@ -197,19 +295,40 @@ def write_image(image_url):
 
     return filename
 
-def create_api():
+def create_api_v1():
     
     auth = tweepy.OAuthHandler(api_key, api_key_secret)
     auth.set_access_token(access_token, access_token_secret)
-    api = tweepy.API(auth)
+    apiv1 = tweepy.API(auth)
     try:
-        api.verify_credentials()
+        apiv1.verify_credentials()
         logged_in = True
     except Exception as e:
         logger.error("Error creating API", exc_info=True)
         raise e
     logger.info("API created")
-    return (api, logged_in)
+    return (apiv1, logged_in)
+
+def create_api_v2():
+
+    try:
+
+        apiv2 = tweepy.Client(
+            consumer_key=api_key,
+            consumer_secret=api_key_secret,
+            access_token=access_token,
+            access_token_secret=access_token_secret
+            )
+        logged_in = True
+
+    except Exception as e:
+
+        logger.error("Error creating API", exc_info=True)
+        raise e
+
+    logger.info("API v2 created")
+
+    return (apiv2, logged_in)
 
 def mastodon():
 
@@ -256,7 +375,6 @@ def twitter_config():
 
     return(api_key, api_key_secret, access_token, access_token_secret)
 
-# Returns the parameter from the specified file
 def get_parameter( parameter, file_path ):
     # Check if secrets file exists
     if not os.path.isfile(file_path):
@@ -273,7 +391,6 @@ def get_parameter( parameter, file_path ):
     print(file_path + "  Missing parameter %s "%parameter)
     sys.exit(0)
 
-###############################################################################
 # main
 
 if __name__ == '__main__':
@@ -319,16 +436,16 @@ if __name__ == '__main__':
                 is_reply = True
                 tweet_id = get_tweet_id(reply_id)
 
-            if len(entry.links) >= 2:
+            if "media_content" in entry:
 
                 with_images = True
                 images_list = []
-                images = len(entry.links) - 1
+                images = len(entry.media_content)
 
                 i = 0
                 while i < images:
 
-                    image_url = entry.links[i+1].href
+                    image_url = entry.media_content[i]['url']
                     image_filename = write_image(image_url)
                     images_list.append(image_filename)
                     i += 1
@@ -337,46 +454,26 @@ if __name__ == '__main__':
 
         if publish:
 
-            tuit_text = get_toot(title)
+            tuit_text, poll_options, options_lst, is_poll = get_toot(title)
 
             print("Tooting...")
             print(tuit_text)
 
             if not logged_in:
 
-                api, logged_in = create_api()
+                apiv1, logged_in = create_api_v1()
 
-            tweet = compose_tweet(tuit_text, with_images, is_reply)
+                apiv2, logged_in = create_api_v2()
 
-            #########################################################
+            if is_poll:
 
-            sql_insert_ids = 'INSERT INTO id(toot_id, tweet_id) VALUES (%s,%s)'
+                tweet = compose_poll(tuit_text, poll_options, options_lst, toot_id)
 
-            conn = None
+            else:
 
-            try:
+                tweet = compose_tweet(tuit_text, with_images, is_reply)
 
-                conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
-
-                cur = conn.cursor()
-
-                cur.execute(sql_insert_ids, (toot_id, tweet.id))
-
-                conn.commit()
-
-                cur.close()
-
-            except (Exception, psycopg2.DatabaseError) as error:
-
-                print(error)
-
-            finally:
-
-                if conn is not None:
-
-                    conn.close()
-
-            #########################################################
+                write_db(toot_id, tweet.id)
 
             time.sleep(2)
 
@@ -384,3 +481,4 @@ if __name__ == '__main__':
 
             print("Any new feeds")
             sys.exit(0)
+

requirements.txt

@@ -1,8 +1,8 @@
 wheel>=0.37.0
-psycopg2>=2.9.1
+psycopg2-binary
 feedparser>=6.0.8
-bs4>=0.0.1
+bs4
 Mastodon.py>=1.5.1
-tweepy==4.1.0
+tweepy>=4.5.0
 filetype>=1.0.8
 ffmpeg-python>=0.2.0