From 19e78342ca18444abc0e5ac055398350dac70017 Mon Sep 17 00:00:00 2001 From: spla Date: Fri, 10 Sep 2021 20:32:11 +0200 Subject: [PATCH] First mastotuit release! --- README.md | 22 +++++ db-setup.py | 146 +++++++++++++++++++++++++++++ mastotuit.py | 235 +++++++++++++++++++++++++++++++++++++++++++++++ requeriments.txt | 20 ++++ setup.py | 51 ++++++++++ 5 files changed, 474 insertions(+) create mode 100644 README.md create mode 100644 db-setup.py create mode 100644 mastotuit.py create mode 100644 requeriments.txt create mode 100644 setup.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..b68717f --- /dev/null +++ b/README.md @@ -0,0 +1,22 @@ +# mastotuit +Publish automagically to Twitter all your Mastodon posts! + +### Dependencies + +- **Python 3** +- Postgresql server +- Mastodon user account +- Your personal Linux driven PC or laptop + +### Usage: + +Within Python Virtual Environment: + +1. Run `pip install -r requirements.txt` to install needed libraries. + +2. Run `python db-setup.py` to setup and create new Postgresql database and needed table in it and setup your Mastodon's account RSS feed in the format 'https://your.mastodon.server/@your_user.rss' + +3. Run `python setup.py` to input and save your Twitter's key and access tokens. You can get your keys and tokens from (Twitter Developer Platform)[https://developer.twitter.com/en/apply/user.html] + +4. Use your favourite scheduling method to set `python mastotuit.py` to run every minute.. + diff --git a/db-setup.py b/db-setup.py new file mode 100644 index 0000000..8b574bf --- /dev/null +++ b/db-setup.py @@ -0,0 +1,146 @@ +import getpass +import os +import sys +import psycopg2 +from psycopg2 import sql +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT + +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, asking."%file_path) + write_parameter( parameter, file_path ) + #sys.exit(0) + + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) + +def write_parameter( parameter, file_path ): + if not os.path.exists('config'): + os.makedirs('config') + print("Setting up mastotuit parameters...") + print("\n") + feeds_db = input("feeds db name: ") + feeds_db_user = input("feeds db user: ") + feeds_url = input("enter Mastodon user RSS feeds url: ") + + with open(file_path, "w") as text_file: + print("feeds_db: {}".format(feeds_db), file=text_file) + print("feeds_db_user: {}".format(feeds_db_user), file=text_file) + print("feeds_url: {}".format(feeds_url), file=text_file) + +def create_table(db, db_user, table, sql): + + conn = None + try: + + conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() + + print("Creating table.. "+table) + # Create the table in PostgreSQL database + cur.execute(sql) + + conn.commit() + print("Table "+table+" created!") + print("\n") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + +############################################################################################# + +# Load configuration from config file +config_filepath = "config/db_config.txt" +feeds_db = get_parameter("feeds_db", config_filepath) +feeds_db_user = get_parameter("feeds_db_user", config_filepath) +feeds_url = get_parameter("feeds_url", config_filepath) + +############################################################ +# create database +############################################################ + +conn = None + +try: + + conn = psycopg2.connect(dbname='postgres', + user=feeds_db_user, host='', + password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print("Creating database " + feeds_db + ". Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format( + sql.Identifier(feeds_db)) + ) + print("Database " + feeds_db + " created!") + +except (Exception, psycopg2.DatabaseError) as error: + + print(error) + +finally: + + if conn is not None: + + conn.close() + +############################################################################################# + +try: + + conn = None + conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + +except (Exception, psycopg2.DatabaseError) as error: + + print(error) + # Load configuration from config file + os.remove("db_config.txt") + print("Exiting. Run db-setup again with right parameters") + sys.exit(0) + +if conn is not None: + + print("\n") + print("mastotuit parameters saved to db-config.txt!") + print("\n") + +############################################################ +# Create needed tables +############################################################ + +print("Creating table...") + +######################################## + +db = feeds_db +db_user = feeds_db_user +table = "feeds" +sql = "create table "+table+" (link varchar(300) PRIMARY KEY)" +create_table(db, db_user, table, sql) + +##################################### + +print("Done!") +print("Now you can run setup.py!") +print("\n") diff --git a/mastotuit.py b/mastotuit.py new file mode 100644 index 0000000..051783c --- /dev/null +++ b/mastotuit.py @@ -0,0 +1,235 @@ +import os +import feedparser +import re +from mastodon import Mastodon +import psycopg2 +import sys +import time +import tweepy +from tweepy import TweepError +import logging +import pdb + +logger = logging.getLogger() + +def cleanhtml(raw_html): + + cleanr = re.compile('<.*?>') + cleantext = re.sub(cleanr, '', raw_html) + return cleantext + +def unescape(s): + + s = s.replace("'", "'") + s = s.replace('"', '"') + return s + +def create_api(): + + auth = tweepy.OAuthHandler(api_key, api_key_secret) + auth.set_access_token(access_token, access_token_secret) + api = tweepy.API(auth, wait_on_rate_limit=True, + wait_on_rate_limit_notify=True) + try: + api.verify_credentials() + except Exception as e: + logger.error("Error creating API", exc_info=True) + raise e + logger.info("API created") + return api + +def mastodon(): + + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) + + # Load configuration from config file + config_filepath = "config/config.txt" + mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) + + # Initialise Mastodon API + mastodon = Mastodon( + client_id=uc_client_id, + client_secret=uc_client_secret, + access_token=uc_access_token, + api_base_url='https://' + mastodon_hostname, + ) + + # Initialise access headers + headers = {'Authorization': 'Bearer %s'%uc_access_token} + + return (mastodon, mastodon_hostname) + +def db_config(): + + # Load db configuration from config file + db_config_filepath = "config/db_config.txt" + feeds_db = get_parameter("feeds_db", db_config_filepath) + feeds_db_user = get_parameter("feeds_db_user", db_config_filepath) + feeds_url = get_parameter("feeds_url", db_config_filepath) + + return (feeds_db, feeds_db_user, feeds_url) + +def twitter_config(): + + twitter_config_filepath = "config/keys_config.txt" + api_key = get_parameter("api_key", twitter_config_filepath) + api_key_secret = get_parameter("api_key_secret", twitter_config_filepath) + access_token = get_parameter("access_token", twitter_config_filepath) + access_token_secret = get_parameter("access_token_secret", twitter_config_filepath) + + return(api_key, api_key_secret, access_token, access_token_secret) + +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, exiting."%file_path) + sys.exit(0) + + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) + +############################################################################### +# main + +if __name__ == '__main__': + + ####################################################################### + #mastodon, mastodon_hostname = mastodon() + + feeds_db, feeds_db_user, feeds_url = db_config() + + api_key, api_key_secret, access_token, access_token_secret = twitter_config() + + api = create_api() + + publish = 0 + + try: + + newsfeeds = feedparser.parse(feeds_url) + + except: + + print(newsfeeds.status) + sys.exit(0) + + for entry in newsfeeds.entries: + + title = entry['summary'] + id = entry['id'] + link = entry['link'] + + ################################################################### + # check database if feed is already published + + try: + + conn = None + conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute('select link from feeds where link=(%s)', (link,)) + + row = cur.fetchone() + if row == None: + publish = 1 + else: + publish = 0 + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ########################################################### + + if publish == 1: + + toot_text = str(title)+'\n' + toot_text += str(link) + + toot_text = cleanhtml(toot_text) + toot_text = unescape(toot_text) + + print("Tooting...") + print(toot_text) + + if len(toot_text) < 280: + + try: + + api.update_status(toot_text) + + except TweepError as err: + + print('\n') + sys.exit(err) + + else: + + toot_text1 = toot_text[:int(len(toot_text)/2)].rsplit(' ', 1)[0] + toot_text2 = toot_text[int(len(toot_text1)):] + + try: + + first_tweet = api.update_status(toot_text1) + api.update_status(toot_text2, in_reply_to_status_id=first_tweet.id) + + except TweepError as err: + + print('\n') + sys.exit(err) + + time.sleep(2) + + ######################################################### + + insert_line = 'INSERT INTO feeds(link) VALUES (%s)' + + conn = None + + try: + + conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(insert_line, (link,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + else: + + print("Any new feeds") + sys.exit(0) diff --git a/requeriments.txt b/requeriments.txt new file mode 100644 index 0000000..d3a9e12 --- /dev/null +++ b/requeriments.txt @@ -0,0 +1,20 @@ +blurhash==1.1.4 +certifi==2021.5.30 +charset-normalizer==2.0.4 +decorator==5.0.9 +feedparser==6.0.8 +idna==3.2 +Mastodon.py==1.5.1 +oauthlib==3.1.1 +psycopg2==2.9.1 +psycopg2-binary==2.9.1 +PySocks==1.7.1 +python-dateutil==2.8.2 +python-magic==0.4.24 +pytz==2021.1 +requests==2.26.0 +requests-oauthlib==1.3.0 +sgmllib3k==1.0.0 +six==1.16.0 +tweepy==3.10.0 +urllib3==1.26.6 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3c063bb --- /dev/null +++ b/setup.py @@ -0,0 +1,51 @@ +import getpass +import os +import sys + +# Returns the parameter from the specified file +def get_parameter( parameter, file_path ): + # Check if secrets file exists + if not os.path.isfile(file_path): + print("File %s not found, asking."%file_path) + write_parameter( parameter, file_path ) + #sys.exit(0) + + # Find parameter in file + with open( file_path ) as f: + for line in f: + if line.startswith( parameter ): + return line.replace(parameter + ":", "").strip() + + # Cannot find parameter, exit + print(file_path + " Missing parameter %s "%parameter) + sys.exit(0) + +def write_parameter( parameter, file_path ): + if not os.path.exists('config'): + os.makedirs('config') + print("Setting up Twitter keys & tokens ...") + print("\n") + api_key = input("API Key: ") + api_key_secret = input("API Key Secret: ") + access_token = input("Access Token: ") + access_token_secret = input("Access Token Secret: ") + + with open(file_path, "w") as text_file: + print("api_key: {}".format(api_key), file=text_file) + print("api_key_secret: {}".format(api_key_secret), file=text_file) + print("access_token: {}".format(access_token), file=text_file) + print("access_token_secret: {}".format(access_token_secret), file=text_file) + + print('\nDone!') + +############################################################################### +# main + +if __name__ == '__main__': + + # Load configuration from config file + twitter_config_filepath = "config/keys_config.txt" + api_key = get_parameter("api_key", twitter_config_filepath) + api_key_secret = get_parameter("api_key_secret", twitter_config_filepath) + access_token = get_parameter("access_token", twitter_config_filepath) + access_token_secret = get_parameter("access_token_secret", twitter_config_filepath)