spla/mastofeeds
1
0
Fork 0
Codi Incidències Sol·licituds Pull Paquets Projectes Llançaments 1 Wiki Activitat

Increased link column to varchar(300)

Explorar la Font
This commit is contained in:
spla 2021-05-15 11:30:59 +02:00
pare f085594065 e3af314a49
commit c9219f1a85
S'han modificat 4 arxius amb 314 adicions i 298 eliminacions
Mostrar estadístiques Descarregar el fitxer de pedaç Descarregar el fitxer de diferències Expand all files Collapse all files

15
README.md
Veure arxiu

@ -7,21 +7,20 @@ Publish to Mastodon's server the rss feed of your choice.
- **Python 3** - **Python 3**
- Postgresql server - Postgresql server
- Everything else at the top of `mastofeeds.py`! - Mastodon's bot account
### Usage: ### Usage:
Within Python Virtual Environment: Within Python Virtual Environment:
1. Run 'python db-setup.py' to create needed database and table. It's needed to control what feeds are already published. db-setup.py 1. Run `pip install -r requirements.txt` to install all needed libraries.
2. Run `python db-setup.py` to create needed database and table. It's needed to control what feeds are already published. db-setup.py
will also ask you the feed's url. will also ask you the feed's url.
2. Run 'python setup.py' to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use. 3. Run `python setup.py` to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use.
3. Run 'python mastofeeds.py' to start publishing feeds. 4. Run `python mastofeeds.py` to start publishing feeds.
4. Use your favourite scheduling method to set mastofeeds.py to run regularly.
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
5. Use your favourite scheduling method to set `python mastofeeds.py` to run regularly.

199
db-setup.py
Veure arxiu

@ -29,121 +29,122 @@ def get_parameter( parameter, file_path ):
sys.exit(0) sys.exit(0)
def write_parameter( parameter, file_path ): def write_parameter( parameter, file_path ):
print("Setting up newsfeed parameters...") print("Setting up newsfeed parameters...")
print("\n") print("\n")
feeds_db = input("feeds db name: ") feeds_db = input("feeds db name: ")
feeds_db_user = input("feeds db user: ") feeds_db_user = input("feeds db user: ")
feeds_url = input("enter feeds url: ") feeds_url = input("enter feeds url: ")
with open(file_path, "w") as text_file: with open(file_path, "w") as text_file:
print("feeds_db: {}".format(feeds_db), file=text_file) print("feeds_db: {}".format(feeds_db), file=text_file)
print("feeds_db_user: {}".format(feeds_db_user), file=text_file) print("feeds_db_user: {}".format(feeds_db_user), file=text_file)
print("feeds_url: {}".format(feeds_url), file=text_file) print("feeds_url: {}".format(feeds_url), file=text_file)
def create_table(db, db_user, table, sql): def create_table(db, db_user, table, sql):
conn = None conn = None
try: try:
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
print("Creating table.. "+table) print("Creating table.. "+table)
# Create the table in PostgreSQL database # Create the table in PostgreSQL database
cur.execute(sql) cur.execute(sql)
conn.commit() conn.commit()
print("Table "+table+" created!") print("Table "+table+" created!")
print("\n") print("\n")
except (Exception, psycopg2.DatabaseError) as error: except (Exception, psycopg2.DatabaseError) as error:
print(error) print(error)
finally: finally:
if conn is not None:
conn.close()
###############################################################################
# main
if __name__ == '__main__':
# Load configuration from config file
config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", config_filepath)
feeds_db_user = get_parameter("feeds_db_user", config_filepath)
feeds_url = get_parameter("feeds_url", config_filepath)
############################################################
# create database
############################################################
conn = None
try:
conn = psycopg2.connect(dbname='postgres', user=feeds_db_user, host='', password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + feeds_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(feeds_db)))
print("Database " + feeds_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None: if conn is not None:
conn.close() print("\n")
print("Host parameters saved to config.txt!")
print("\n")
############################################################################################# ############################################################
# Create needed tables
############################################################
# Load configuration from config file print("Creating table...")
config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", config_filepath)
feeds_db_user = get_parameter("feeds_db_user", config_filepath)
feeds_url = get_parameter("feeds_url", config_filepath)
############################################################ #####################################
# create database
############################################################
conn = None db = feeds_db
db_user = feeds_db_user
table = "feeds"
sql = "create table "+table+" (link varchar(300) PRIMARY KEY)"
create_table(db, db_user, table, sql)
try: #####################################
conn = psycopg2.connect(dbname='postgres', print("Done!")
user=feeds_db_user, host='', print("Now you can run setup.py!")
password='') print("\n")
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + feeds_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(feeds_db))
)
print("Database " + feeds_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None:
print("\n")
print("Host parameters saved to config.txt!")
print("\n")
############################################################
# Create needed tables
############################################################
print("Creating table...")
########################################
db = feeds_db
db_user = feeds_db_user
table = "feeds"
sql = "create table "+table+" (link varchar(300) PRIMARY KEY)"
create_table(db, db_user, table, sql)
#####################################
print("Done!")
print("Now you can run setup.py!")
print("\n")

170
mastofeeds.py
Veure arxiu

@ -3,10 +3,7 @@ import feedparser
from mastodon import Mastodon from mastodon import Mastodon
import psycopg2 import psycopg2
import sys import sys
import time
###############################################################################
# INITIALISATION
###############################################################################
# Returns the parameter from the specified file # Returns the parameter from the specified file
def get_parameter( parameter, file_path ): def get_parameter( parameter, file_path ):
@ -25,103 +22,75 @@ def get_parameter( parameter, file_path ):
print(file_path + " Missing parameter %s "%parameter) print(file_path + " Missing parameter %s "%parameter)
sys.exit(0) sys.exit(0)
# Load secrets from secrets file ###############################################################################
secrets_filepath = "secrets/secrets.txt" # main
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file if __name__ == '__main__':
db_config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", db_config_filepath)
feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
feeds_url = get_parameter("feeds_url", db_config_filepath)
# Load configuration from config file # Load secrets from secrets file
config_filepath = "config.txt" secrets_filepath = "secrets/secrets.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Initialise Mastodon API # Load configuration from config file
mastodon = Mastodon( db_config_filepath = "db_config.txt"
client_id = uc_client_id, feeds_db = get_parameter("feeds_db", db_config_filepath)
client_secret = uc_client_secret, feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
access_token = uc_access_token, feeds_url = get_parameter("feeds_url", db_config_filepath)
api_base_url = 'https://' + mastodon_hostname,
)
# Initialise access headers # Load configuration from config file
headers={ 'Authorization': 'Bearer %s'%uc_access_token } config_filepath = "config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social
######################################################## # Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
)
publish = 0 # Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
newsfeeds = feedparser.parse(feeds_url) #######################################################################
for entry in newsfeeds.entries: publish = 0
title = entry['title']
id = entry['id']
link = entry['link']
###################################################################
# check database if feed is already published
###################################################################
try: try:
conn = None newsfeeds = feedparser.parse(feeds_url)
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") print(newsfeeds.status)
cur = conn.cursor() except:
cur.execute('select link from feeds where link=(%s)', (link,)) print(newsfeeds.status)
sys.exit(0)
row = cur.fetchone() for entry in newsfeeds.entries:
if row == None:
publish = 1
else:
publish = 0
cur.close() title = entry['title']
id = entry['id']
link = entry['link']
except (Exception, psycopg2.DatabaseError) as error: ###################################################################
# check database if feed is already published
print(error)
finally:
if conn is not None:
conn.close()
###########################################################
if publish == 1:
toot_text = str(title)+'\n'
toot_text += str(link)
print("Tooting...")
print(toot_text)
mastodon.status_post(toot_text, in_reply_to_id=None,)
#########################################################
insert_line = 'INSERT INTO feeds(link) VALUES (%s)'
conn = None
try: try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432") conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor() cur = conn.cursor()
cur.execute(insert_line, (link,)) cur.execute('select id from feeds where id=(%s)', (id,))
conn.commit() row = cur.fetchone()
if row == None:
publish = 1
else:
publish = 0
cur.close() cur.close()
@ -134,6 +103,49 @@ for entry in newsfeeds.entries:
if conn is not None: if conn is not None:
conn.close() conn.close()
else:
print("Any new feeds") ###########################################################
if publish == 1:
toot_text = str(title)+'\n'
toot_text += str(link)
print("Tooting...")
print(toot_text)
mastodon.status_post(toot_text, in_reply_to_id=None,)
time.sleep(2)
#########################################################
insert_line = 'INSERT INTO feeds(id, link) VALUES (%s, %s)'
conn = None
try:
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_line, (id, link,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
else:
print("Any new feeds")
sys.exit(0)

228
setup.py
Veure arxiu

@ -9,126 +9,125 @@ import os
import sys import sys
def create_dir(): def create_dir():
if not os.path.exists('secrets'): if not os.path.exists('secrets'):
os.makedirs('secrets') os.makedirs('secrets')
def create_file(): def create_file():
if not os.path.exists('secrets/secrets.txt'): if not os.path.exists('secrets/secrets.txt'):
with open('secrets/secrets.txt', 'w'): pass with open('secrets/secrets.txt', 'w'): pass
print(secrets_filepath + " created!") print(secrets_filepath + " created!")
def create_config(): def create_config():
if not os.path.exists(config_filepath): if not os.path.exists(config_filepath):
print(config_filepath + " created!") print(config_filepath + " created!")
with open('config.txt', 'w'): pass with open('config.txt', 'w'): pass
def write_params(): def write_params():
with open(secrets_filepath, 'a') as the_file: with open(secrets_filepath, 'a') as the_file:
print("Writing secrets parameter names to " + secrets_filepath) print("Writing secrets parameter names to " + secrets_filepath)
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n') the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
def write_config(): def write_config():
with open(config_filepath, 'a') as the_file: with open(config_filepath, 'a') as the_file:
the_file.write('mastodon_hostname: \n') the_file.write('mastodon_hostname: \n')
print("adding parameter name 'mastodon_hostname' to "+ config_filepath) print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
def read_client_lines(self): def read_client_lines(self):
client_path = 'app_clientcred.txt' client_path = 'app_clientcred.txt'
with open(client_path) as fp: with open(client_path) as fp:
line = fp.readline() line = fp.readline()
cnt = 1 cnt = 1
while line: while line:
if cnt == 1: if cnt == 1:
print("Writing client id to " + secrets_filepath) print("Writing client id to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip()) modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
elif cnt == 2: elif cnt == 2:
print("Writing client secret to " + secrets_filepath) print("Writing client secret to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip()) modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
line = fp.readline() line = fp.readline()
cnt += 1 cnt += 1
def read_token_line(self): def read_token_line(self):
token_path = 'app_usercred.txt' token_path = 'app_usercred.txt'
with open(token_path) as fp: with open(token_path) as fp:
line = fp.readline() line = fp.readline()
print("Writing access token to " + secrets_filepath) print("Writing access token to " + secrets_filepath)
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip()) modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
def read_config_line(): def read_config_line():
with open(config_filepath) as fp: with open(config_filepath) as fp:
line = fp.readline() line = fp.readline()
modify_file(config_filepath, "mastodon_hostname: ", value=hostname) modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
def log_in(): def log_in():
error = 0 error = 0
try: try:
global hostname global hostname
hostname = input("Enter Mastodon hostname: ") hostname = input("Enter Mastodon hostname: ")
user_name = input("User name, ex. user@" + hostname +"? ") user_name = input("User name, ex. user@" + hostname +"? ")
user_password = getpass.getpass("User password? ") user_password = getpass.getpass("User password? ")
app_name = input("This app name? ") app_name = input("This app name? ")
Mastodon.create_app(app_name, scopes=["read","write"], Mastodon.create_app(app_name, scopes=["read","write"],
to_file="app_clientcred.txt", api_base_url=hostname) to_file="app_clientcred.txt", api_base_url=hostname)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname) mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
mastodon.log_in( mastodon.log_in(
user_name, user_name,
user_password, user_password,
scopes = ["read", "write"], scopes = ["read", "write"],
to_file = "app_usercred.txt" to_file = "app_usercred.txt"
) )
except MastodonIllegalArgumentError as i_error: except MastodonIllegalArgumentError as i_error:
error = 1 error = 1
if os.path.exists("secrets/secrets.txt"): if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..") print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt") os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"): if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..") print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt") os.remove("app_clientcred.txt")
sys.exit(i_error) sys.exit(i_error)
except MastodonNetworkError as n_error: except MastodonNetworkError as n_error:
error = 1 error = 1
if os.path.exists("secrets/secrets.txt"): if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..") print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt") os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"): if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..") print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt") os.remove("app_clientcred.txt")
sys.exit(n_error) sys.exit(n_error)
except MastodonReadTimeout as r_error: except MastodonReadTimeout as r_error:
error = 1 error = 1
if os.path.exists("secrets/secrets.txt"): if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..") print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt") os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"): if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..") print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt") os.remove("app_clientcred.txt")
sys.exit(r_error) sys.exit(r_error)
except MastodonAPIError as a_error: except MastodonAPIError as a_error:
error = 1 error = 1
if os.path.exists("secrets/secrets.txt"): if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..") print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt") os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"): if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..") print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt") os.remove("app_clientcred.txt")
sys.exit(a_error) sys.exit(a_error)
finally: finally:
if error == 0: if error == 0:
create_dir()
create_dir() create_file()
create_file() write_params()
write_params() client_path = 'app_clientcred.txt'
client_path = 'app_clientcred.txt' read_client_lines(client_path)
read_client_lines(client_path) token_path = 'app_usercred.txt'
token_path = 'app_usercred.txt' read_token_line(token_path)
read_token_line(token_path) if os.path.exists("app_clientcred.txt"):
if os.path.exists("app_clientcred.txt"): print("Removing app_clientcred.txt temp file..")
print("Removing app_clientcred.txt temp file..") os.remove("app_clientcred.txt")
os.remove("app_clientcred.txt") if os.path.exists("app_usercred.txt"):
if os.path.exists("app_usercred.txt"): print("Removing app_usercred.txt temp file..")
print("Removing app_usercred.txt temp file..") os.remove("app_usercred.txt")
os.remove("app_usercred.txt") print("Secrets setup done!\n")
print("Secrets setup done!\n")
def modify_file(file_name,pattern,value=""): def modify_file(file_name,pattern,value=""):
fh=fileinput.input(file_name,inplace=True) fh=fileinput.input(file_name,inplace=True)
@ -175,12 +174,17 @@ def get_hostname( parameter, config_filepath ):
print("hostname setup done!") print("hostname setup done!")
sys.exit(0) sys.exit(0)
# Load secrets from secrets file ###############################################################################
secrets_filepath = "secrets/secrets.txt" # main
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file if __name__ == '__main__':
config_filepath = "config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) # E.g., mastodon.social # Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)