diff --git a/README.md b/README.md index f521531..5d5c3fd 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,20 @@ Publish to Mastodon's server the rss feed of your choice. - **Python 3** - Postgresql server -- Everything else at the top of `mastofeeds.py`! +- Mastodon's bot account ### Usage: Within Python Virtual Environment: -1. Run 'python db-setup.py' to create needed database and table. It's needed to control what feeds are already published. db-setup.py +1. Run `pip install -r requirements.txt` to install all needed libraries. + +2. Run `python db-setup.py` to create needed database and table. It's needed to control what feeds are already published. db-setup.py will also ask you the feed's url. -2. Run 'python setup.py' to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use. +3. Run `python setup.py` to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use. -3. Run 'python mastofeeds.py' to start publishing feeds. - -4. Use your favourite scheduling method to set mastofeeds.py to run regularly. - -Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them. +4. Run `python mastofeeds.py` to start publishing feeds. +5. Use your favourite scheduling method to set `python mastofeeds.py` to run regularly. diff --git a/db-setup.py b/db-setup.py index b460446..111dad4 100644 --- a/db-setup.py +++ b/db-setup.py @@ -29,121 +29,122 @@ def get_parameter( parameter, file_path ): sys.exit(0) def write_parameter( parameter, file_path ): - print("Setting up newsfeed parameters...") - print("\n") - feeds_db = input("feeds db name: ") - feeds_db_user = input("feeds db user: ") - feeds_url = input("enter feeds url: ") + print("Setting up newsfeed parameters...") + print("\n") + feeds_db = input("feeds db name: ") + feeds_db_user = input("feeds db user: ") + feeds_url = input("enter feeds url: ") - with open(file_path, "w") as text_file: - print("feeds_db: {}".format(feeds_db), file=text_file) - print("feeds_db_user: {}".format(feeds_db_user), file=text_file) - print("feeds_url: {}".format(feeds_url), file=text_file) + with open(file_path, "w") as text_file: + print("feeds_db: {}".format(feeds_db), file=text_file) + print("feeds_db_user: {}".format(feeds_db_user), file=text_file) + print("feeds_url: {}".format(feeds_url), file=text_file) def create_table(db, db_user, table, sql): - conn = None - try: + conn = None + try: - conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") - cur = conn.cursor() + conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") + cur = conn.cursor() - print("Creating table.. "+table) - # Create the table in PostgreSQL database - cur.execute(sql) + print("Creating table.. "+table) + # Create the table in PostgreSQL database + cur.execute(sql) - conn.commit() - print("Table "+table+" created!") - print("\n") + conn.commit() + print("Table "+table+" created!") + print("\n") - except (Exception, psycopg2.DatabaseError) as error: + except (Exception, psycopg2.DatabaseError) as error: - print(error) + print(error) - finally: + finally: + + if conn is not None: + + conn.close() + +############################################################################### +# main + +if __name__ == '__main__': + + # Load configuration from config file + config_filepath = "db_config.txt" + feeds_db = get_parameter("feeds_db", config_filepath) + feeds_db_user = get_parameter("feeds_db_user", config_filepath) + feeds_url = get_parameter("feeds_url", config_filepath) + + ############################################################ + # create database + ############################################################ + + conn = None + + try: + + conn = psycopg2.connect(dbname='postgres', user=feeds_db_user, host='', password='') + + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) + + cur = conn.cursor() + + print("Creating database " + feeds_db + ". Please wait...") + + cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(feeds_db))) + + print("Database " + feeds_db + " created!") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + + ############################################################################################# + + try: + + conn = None + conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + # Load configuration from config file + os.remove("db_config.txt") + print("Exiting. Run setup again with right parameters") + sys.exit(0) if conn is not None: - conn.close() + print("\n") + print("Host parameters saved to config.txt!") + print("\n") -############################################################################################# + ############################################################ + # Create needed tables + ############################################################ -# Load configuration from config file -config_filepath = "db_config.txt" -feeds_db = get_parameter("feeds_db", config_filepath) -feeds_db_user = get_parameter("feeds_db_user", config_filepath) -feeds_url = get_parameter("feeds_url", config_filepath) + print("Creating table...") -############################################################ -# create database -############################################################ + ##################################### -conn = None + db = feeds_db + db_user = feeds_db_user + table = "feeds" + sql = "create table "+table+" (link varchar(300) PRIMARY KEY)" + create_table(db, db_user, table, sql) -try: + ##################################### - conn = psycopg2.connect(dbname='postgres', - user=feeds_db_user, host='', - password='') - - conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) - - cur = conn.cursor() - - print("Creating database " + feeds_db + ". Please wait...") - - cur.execute(sql.SQL("CREATE DATABASE {}").format( - sql.Identifier(feeds_db)) - ) - print("Database " + feeds_db + " created!") - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - -finally: - - if conn is not None: - - conn.close() - -############################################################################################# - -try: - - conn = None - conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") - -except (Exception, psycopg2.DatabaseError) as error: - - print(error) - # Load configuration from config file - os.remove("db_config.txt") - print("Exiting. Run setup again with right parameters") - sys.exit(0) - -if conn is not None: - - print("\n") - print("Host parameters saved to config.txt!") - print("\n") - -############################################################ -# Create needed tables -############################################################ - -print("Creating table...") - -######################################## - -db = feeds_db -db_user = feeds_db_user -table = "feeds" -sql = "create table "+table+" (link varchar(300) PRIMARY KEY)" -create_table(db, db_user, table, sql) - -##################################### - -print("Done!") -print("Now you can run setup.py!") -print("\n") + print("Done!") + print("Now you can run setup.py!") + print("\n") diff --git a/mastofeeds.py b/mastofeeds.py index 4b6e8c1..d4d0d2c 100644 --- a/mastofeeds.py +++ b/mastofeeds.py @@ -3,10 +3,7 @@ import feedparser from mastodon import Mastodon import psycopg2 import sys - -############################################################################### -# INITIALISATION -############################################################################### +import time # Returns the parameter from the specified file def get_parameter( parameter, file_path ): @@ -25,103 +22,75 @@ def get_parameter( parameter, file_path ): print(file_path + " Missing parameter %s "%parameter) sys.exit(0) -# Load secrets from secrets file -secrets_filepath = "secrets/secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) +############################################################################### +# main -# Load configuration from config file -db_config_filepath = "db_config.txt" -feeds_db = get_parameter("feeds_db", db_config_filepath) -feeds_db_user = get_parameter("feeds_db_user", db_config_filepath) -feeds_url = get_parameter("feeds_url", db_config_filepath) +if __name__ == '__main__': -# Load configuration from config file -config_filepath = "config.txt" -mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) -# Initialise Mastodon API -mastodon = Mastodon( - client_id = uc_client_id, - client_secret = uc_client_secret, - access_token = uc_access_token, - api_base_url = 'https://' + mastodon_hostname, -) + # Load configuration from config file + db_config_filepath = "db_config.txt" + feeds_db = get_parameter("feeds_db", db_config_filepath) + feeds_db_user = get_parameter("feeds_db_user", db_config_filepath) + feeds_url = get_parameter("feeds_url", db_config_filepath) -# Initialise access headers -headers={ 'Authorization': 'Bearer %s'%uc_access_token } + # Load configuration from config file + config_filepath = "config.txt" + mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social -######################################################## + # Initialise Mastodon API + mastodon = Mastodon( + client_id = uc_client_id, + client_secret = uc_client_secret, + access_token = uc_access_token, + api_base_url = 'https://' + mastodon_hostname, + ) -publish = 0 + # Initialise access headers + headers={ 'Authorization': 'Bearer %s'%uc_access_token } -newsfeeds = feedparser.parse(feeds_url) + ####################################################################### -for entry in newsfeeds.entries: - - title = entry['title'] - id = entry['id'] - link = entry['link'] - - ################################################################### - # check database if feed is already published - ################################################################### + publish = 0 try: - conn = None - conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + newsfeeds = feedparser.parse(feeds_url) + print(newsfeeds.status) - cur = conn.cursor() + except: - cur.execute('select link from feeds where link=(%s)', (link,)) + print(newsfeeds.status) + sys.exit(0) - row = cur.fetchone() - if row == None: - publish = 1 - else: - publish = 0 + for entry in newsfeeds.entries: - cur.close() + title = entry['title'] + id = entry['id'] + link = entry['link'] - except (Exception, psycopg2.DatabaseError) as error: - - print(error) - - finally: - - if conn is not None: - - conn.close() - - ########################################################### - - if publish == 1: - - toot_text = str(title)+'\n' - toot_text += str(link) - - print("Tooting...") - print(toot_text) - - mastodon.status_post(toot_text, in_reply_to_id=None,) - - ######################################################### - - insert_line = 'INSERT INTO feeds(link) VALUES (%s)' - - conn = None + ################################################################### + # check database if feed is already published try: + conn = None conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") cur = conn.cursor() - cur.execute(insert_line, (link,)) + cur.execute('select id from feeds where id=(%s)', (id,)) - conn.commit() + row = cur.fetchone() + if row == None: + publish = 1 + else: + publish = 0 cur.close() @@ -134,6 +103,49 @@ for entry in newsfeeds.entries: if conn is not None: conn.close() - else: - print("Any new feeds") + ########################################################### + + if publish == 1: + + toot_text = str(title)+'\n' + toot_text += str(link) + + print("Tooting...") + print(toot_text) + + mastodon.status_post(toot_text, in_reply_to_id=None,) + + time.sleep(2) + + ######################################################### + + insert_line = 'INSERT INTO feeds(id, link) VALUES (%s, %s)' + + conn = None + + try: + + conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") + + cur = conn.cursor() + + cur.execute(insert_line, (id, link,)) + + conn.commit() + + cur.close() + + except (Exception, psycopg2.DatabaseError) as error: + + print(error) + + finally: + + if conn is not None: + + conn.close() + else: + + print("Any new feeds") + sys.exit(0) diff --git a/setup.py b/setup.py index f32d45b..8e6303d 100644 --- a/setup.py +++ b/setup.py @@ -9,126 +9,125 @@ import os import sys def create_dir(): - if not os.path.exists('secrets'): - os.makedirs('secrets') + if not os.path.exists('secrets'): + os.makedirs('secrets') def create_file(): - if not os.path.exists('secrets/secrets.txt'): - with open('secrets/secrets.txt', 'w'): pass - print(secrets_filepath + " created!") + if not os.path.exists('secrets/secrets.txt'): + with open('secrets/secrets.txt', 'w'): pass + print(secrets_filepath + " created!") def create_config(): - if not os.path.exists(config_filepath): - print(config_filepath + " created!") - with open('config.txt', 'w'): pass + if not os.path.exists(config_filepath): + print(config_filepath + " created!") + with open('config.txt', 'w'): pass def write_params(): - with open(secrets_filepath, 'a') as the_file: - print("Writing secrets parameter names to " + secrets_filepath) - the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') + with open(secrets_filepath, 'a') as the_file: + print("Writing secrets parameter names to " + secrets_filepath) + the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') def write_config(): - with open(config_filepath, 'a') as the_file: - the_file.write('mastodon_hostname: \n') - print("adding parameter name 'mastodon_hostname' to "+ config_filepath) + with open(config_filepath, 'a') as the_file: + the_file.write('mastodon_hostname: \n') + print("adding parameter name 'mastodon_hostname' to "+ config_filepath) def read_client_lines(self): - client_path = 'app_clientcred.txt' - with open(client_path) as fp: - line = fp.readline() - cnt = 1 - while line: - if cnt == 1: - print("Writing client id to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) - elif cnt == 2: - print("Writing client secret to " + secrets_filepath) - modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) - line = fp.readline() - cnt += 1 + client_path = 'app_clientcred.txt' + with open(client_path) as fp: + line = fp.readline() + cnt = 1 + while line: + if cnt == 1: + print("Writing client id to " + secrets_filepath) + modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) + elif cnt == 2: + print("Writing client secret to " + secrets_filepath) + modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) + line = fp.readline() + cnt += 1 def read_token_line(self): - token_path = 'app_usercred.txt' - with open(token_path) as fp: - line = fp.readline() - print("Writing access token to " + secrets_filepath) - modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) + token_path = 'app_usercred.txt' + with open(token_path) as fp: + line = fp.readline() + print("Writing access token to " + secrets_filepath) + modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) def read_config_line(): - with open(config_filepath) as fp: - line = fp.readline() - modify_file(config_filepath, "mastodon_hostname: ", value=hostname) + with open(config_filepath) as fp: + line = fp.readline() + modify_file(config_filepath, "mastodon_hostname: ", value=hostname) def log_in(): - error = 0 - try: - global hostname - hostname = input("Enter Mastodon hostname: ") - user_name = input("User name, ex. user@" + hostname +"? ") - user_password = getpass.getpass("User password? ") - app_name = input("This app name? ") - Mastodon.create_app(app_name, scopes=["read","write"], - to_file="app_clientcred.txt", api_base_url=hostname) - mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) - mastodon.log_in( - user_name, - user_password, - scopes = ["read", "write"], - to_file = "app_usercred.txt" - ) - except MastodonIllegalArgumentError as i_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(i_error) - except MastodonNetworkError as n_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(n_error) - except MastodonReadTimeout as r_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(r_error) - except MastodonAPIError as a_error: - error = 1 - if os.path.exists("secrets/secrets.txt"): - print("Removing secrets/secrets.txt file..") - os.remove("secrets/secrets.txt") - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt file..") - os.remove("app_clientcred.txt") - sys.exit(a_error) - finally: - if error == 0: - - create_dir() - create_file() - write_params() - client_path = 'app_clientcred.txt' - read_client_lines(client_path) - token_path = 'app_usercred.txt' - read_token_line(token_path) - if os.path.exists("app_clientcred.txt"): - print("Removing app_clientcred.txt temp file..") - os.remove("app_clientcred.txt") - if os.path.exists("app_usercred.txt"): - print("Removing app_usercred.txt temp file..") - os.remove("app_usercred.txt") - print("Secrets setup done!\n") + error = 0 + try: + global hostname + hostname = input("Enter Mastodon hostname: ") + user_name = input("User name, ex. user@" + hostname +"? ") + user_password = getpass.getpass("User password? ") + app_name = input("This app name? ") + Mastodon.create_app(app_name, scopes=["read","write"], + to_file="app_clientcred.txt", api_base_url=hostname) + mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) + mastodon.log_in( + user_name, + user_password, + scopes = ["read", "write"], + to_file = "app_usercred.txt" + ) + except MastodonIllegalArgumentError as i_error: + error = 1 + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(i_error) + except MastodonNetworkError as n_error: + error = 1 + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(n_error) + except MastodonReadTimeout as r_error: + error = 1 + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(r_error) + except MastodonAPIError as a_error: + error = 1 + if os.path.exists("secrets/secrets.txt"): + print("Removing secrets/secrets.txt file..") + os.remove("secrets/secrets.txt") + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt file..") + os.remove("app_clientcred.txt") + sys.exit(a_error) + finally: + if error == 0: + create_dir() + create_file() + write_params() + client_path = 'app_clientcred.txt' + read_client_lines(client_path) + token_path = 'app_usercred.txt' + read_token_line(token_path) + if os.path.exists("app_clientcred.txt"): + print("Removing app_clientcred.txt temp file..") + os.remove("app_clientcred.txt") + if os.path.exists("app_usercred.txt"): + print("Removing app_usercred.txt temp file..") + os.remove("app_usercred.txt") + print("Secrets setup done!\n") def modify_file(file_name,pattern,value=""): fh=fileinput.input(file_name,inplace=True) @@ -175,12 +174,17 @@ def get_hostname( parameter, config_filepath ): print("hostname setup done!") sys.exit(0) -# Load secrets from secrets file -secrets_filepath = "secrets/secrets.txt" -uc_client_id = get_parameter("uc_client_id", secrets_filepath) -uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) -uc_access_token = get_parameter("uc_access_token", secrets_filepath) +############################################################################### +# main -# Load configuration from config file -config_filepath = "config.txt" -mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) # E.g., mastodon.social +if __name__ == '__main__': + + # Load secrets from secrets file + secrets_filepath = "secrets/secrets.txt" + uc_client_id = get_parameter("uc_client_id", secrets_filepath) + uc_client_secret = get_parameter("uc_client_secret", secrets_filepath) + uc_access_token = get_parameter("uc_access_token", secrets_filepath) + + # Load configuration from config file + config_filepath = "config.txt" + mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)