Increased link column to varchar(300)

This commit is contained in:
spla 2021-05-15 11:30:59 +02:00
commit c9219f1a85
S'han modificat 4 arxius amb 314 adicions i 298 eliminacions

Veure arxiu

@ -7,21 +7,20 @@ Publish to Mastodon's server the rss feed of your choice.
- **Python 3**
- Postgresql server
- Everything else at the top of `mastofeeds.py`!
- Mastodon's bot account
### Usage:
Within Python Virtual Environment:
1. Run 'python db-setup.py' to create needed database and table. It's needed to control what feeds are already published. db-setup.py
1. Run `pip install -r requirements.txt` to install all needed libraries.
2. Run `python db-setup.py` to create needed database and table. It's needed to control what feeds are already published. db-setup.py
will also ask you the feed's url.
2. Run 'python setup.py' to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use.
3. Run `python setup.py` to get your bot's access token of an existing user. It will be saved to 'secrets/secrets.txt' for further use.
3. Run 'python mastofeeds.py' to start publishing feeds.
4. Use your favourite scheduling method to set mastofeeds.py to run regularly.
Note: install all needed packages with 'pip install package' or use 'pip install -r requirements.txt' to install them.
4. Run `python mastofeeds.py` to start publishing feeds.
5. Use your favourite scheduling method to set `python mastofeeds.py` to run regularly.

Veure arxiu

@ -29,121 +29,122 @@ def get_parameter( parameter, file_path ):
sys.exit(0)
def write_parameter( parameter, file_path ):
print("Setting up newsfeed parameters...")
print("\n")
feeds_db = input("feeds db name: ")
feeds_db_user = input("feeds db user: ")
feeds_url = input("enter feeds url: ")
print("Setting up newsfeed parameters...")
print("\n")
feeds_db = input("feeds db name: ")
feeds_db_user = input("feeds db user: ")
feeds_url = input("enter feeds url: ")
with open(file_path, "w") as text_file:
print("feeds_db: {}".format(feeds_db), file=text_file)
print("feeds_db_user: {}".format(feeds_db_user), file=text_file)
print("feeds_url: {}".format(feeds_url), file=text_file)
with open(file_path, "w") as text_file:
print("feeds_db: {}".format(feeds_db), file=text_file)
print("feeds_db_user: {}".format(feeds_db_user), file=text_file)
print("feeds_url: {}".format(feeds_url), file=text_file)
def create_table(db, db_user, table, sql):
conn = None
try:
conn = None
try:
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
conn = psycopg2.connect(database = db, user = db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
print("Creating table.. "+table)
# Create the table in PostgreSQL database
cur.execute(sql)
print("Creating table.. "+table)
# Create the table in PostgreSQL database
cur.execute(sql)
conn.commit()
print("Table "+table+" created!")
print("\n")
conn.commit()
print("Table "+table+" created!")
print("\n")
except (Exception, psycopg2.DatabaseError) as error:
except (Exception, psycopg2.DatabaseError) as error:
print(error)
print(error)
finally:
finally:
if conn is not None:
conn.close()
###############################################################################
# main
if __name__ == '__main__':
# Load configuration from config file
config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", config_filepath)
feeds_db_user = get_parameter("feeds_db_user", config_filepath)
feeds_url = get_parameter("feeds_url", config_filepath)
############################################################
# create database
############################################################
conn = None
try:
conn = psycopg2.connect(dbname='postgres', user=feeds_db_user, host='', password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + feeds_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(feeds_db)))
print("Database " + feeds_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None:
conn.close()
print("\n")
print("Host parameters saved to config.txt!")
print("\n")
#############################################################################################
############################################################
# Create needed tables
############################################################
# Load configuration from config file
config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", config_filepath)
feeds_db_user = get_parameter("feeds_db_user", config_filepath)
feeds_url = get_parameter("feeds_url", config_filepath)
print("Creating table...")
############################################################
# create database
############################################################
#####################################
conn = None
db = feeds_db
db_user = feeds_db_user
table = "feeds"
sql = "create table "+table+" (link varchar(300) PRIMARY KEY)"
create_table(db, db_user, table, sql)
try:
#####################################
conn = psycopg2.connect(dbname='postgres',
user=feeds_db_user, host='',
password='')
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
cur = conn.cursor()
print("Creating database " + feeds_db + ". Please wait...")
cur.execute(sql.SQL("CREATE DATABASE {}").format(
sql.Identifier(feeds_db))
)
print("Database " + feeds_db + " created!")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
#############################################################################################
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
except (Exception, psycopg2.DatabaseError) as error:
print(error)
# Load configuration from config file
os.remove("db_config.txt")
print("Exiting. Run setup again with right parameters")
sys.exit(0)
if conn is not None:
print("\n")
print("Host parameters saved to config.txt!")
print("\n")
############################################################
# Create needed tables
############################################################
print("Creating table...")
########################################
db = feeds_db
db_user = feeds_db_user
table = "feeds"
sql = "create table "+table+" (link varchar(300) PRIMARY KEY)"
create_table(db, db_user, table, sql)
#####################################
print("Done!")
print("Now you can run setup.py!")
print("\n")
print("Done!")
print("Now you can run setup.py!")
print("\n")

Veure arxiu

@ -3,10 +3,7 @@ import feedparser
from mastodon import Mastodon
import psycopg2
import sys
###############################################################################
# INITIALISATION
###############################################################################
import time
# Returns the parameter from the specified file
def get_parameter( parameter, file_path ):
@ -25,103 +22,75 @@ def get_parameter( parameter, file_path ):
print(file_path + " Missing parameter %s "%parameter)
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
###############################################################################
# main
# Load configuration from config file
db_config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", db_config_filepath)
feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
feeds_url = get_parameter("feeds_url", db_config_filepath)
if __name__ == '__main__':
# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
)
# Load configuration from config file
db_config_filepath = "db_config.txt"
feeds_db = get_parameter("feeds_db", db_config_filepath)
feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
feeds_url = get_parameter("feeds_url", db_config_filepath)
# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath) # E.g., mastodon.social
########################################################
# Initialise Mastodon API
mastodon = Mastodon(
client_id = uc_client_id,
client_secret = uc_client_secret,
access_token = uc_access_token,
api_base_url = 'https://' + mastodon_hostname,
)
publish = 0
# Initialise access headers
headers={ 'Authorization': 'Bearer %s'%uc_access_token }
newsfeeds = feedparser.parse(feeds_url)
#######################################################################
for entry in newsfeeds.entries:
title = entry['title']
id = entry['id']
link = entry['link']
###################################################################
# check database if feed is already published
###################################################################
publish = 0
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
newsfeeds = feedparser.parse(feeds_url)
print(newsfeeds.status)
cur = conn.cursor()
except:
cur.execute('select link from feeds where link=(%s)', (link,))
print(newsfeeds.status)
sys.exit(0)
row = cur.fetchone()
if row == None:
publish = 1
else:
publish = 0
for entry in newsfeeds.entries:
cur.close()
title = entry['title']
id = entry['id']
link = entry['link']
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
###########################################################
if publish == 1:
toot_text = str(title)+'\n'
toot_text += str(link)
print("Tooting...")
print(toot_text)
mastodon.status_post(toot_text, in_reply_to_id=None,)
#########################################################
insert_line = 'INSERT INTO feeds(link) VALUES (%s)'
conn = None
###################################################################
# check database if feed is already published
try:
conn = None
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_line, (link,))
cur.execute('select id from feeds where id=(%s)', (id,))
conn.commit()
row = cur.fetchone()
if row == None:
publish = 1
else:
publish = 0
cur.close()
@ -134,6 +103,49 @@ for entry in newsfeeds.entries:
if conn is not None:
conn.close()
else:
print("Any new feeds")
###########################################################
if publish == 1:
toot_text = str(title)+'\n'
toot_text += str(link)
print("Tooting...")
print(toot_text)
mastodon.status_post(toot_text, in_reply_to_id=None,)
time.sleep(2)
#########################################################
insert_line = 'INSERT INTO feeds(id, link) VALUES (%s, %s)'
conn = None
try:
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
cur = conn.cursor()
cur.execute(insert_line, (id, link,))
conn.commit()
cur.close()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
else:
print("Any new feeds")
sys.exit(0)

228
setup.py
Veure arxiu

@ -9,126 +9,125 @@ import os
import sys
def create_dir():
if not os.path.exists('secrets'):
os.makedirs('secrets')
if not os.path.exists('secrets'):
os.makedirs('secrets')
def create_file():
if not os.path.exists('secrets/secrets.txt'):
with open('secrets/secrets.txt', 'w'): pass
print(secrets_filepath + " created!")
if not os.path.exists('secrets/secrets.txt'):
with open('secrets/secrets.txt', 'w'): pass
print(secrets_filepath + " created!")
def create_config():
if not os.path.exists(config_filepath):
print(config_filepath + " created!")
with open('config.txt', 'w'): pass
if not os.path.exists(config_filepath):
print(config_filepath + " created!")
with open('config.txt', 'w'): pass
def write_params():
with open(secrets_filepath, 'a') as the_file:
print("Writing secrets parameter names to " + secrets_filepath)
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
with open(secrets_filepath, 'a') as the_file:
print("Writing secrets parameter names to " + secrets_filepath)
the_file.write('uc_client_id: \n'+'uc_client_secret: \n'+'uc_access_token: \n')
def write_config():
with open(config_filepath, 'a') as the_file:
the_file.write('mastodon_hostname: \n')
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
with open(config_filepath, 'a') as the_file:
the_file.write('mastodon_hostname: \n')
print("adding parameter name 'mastodon_hostname' to "+ config_filepath)
def read_client_lines(self):
client_path = 'app_clientcred.txt'
with open(client_path) as fp:
line = fp.readline()
cnt = 1
while line:
if cnt == 1:
print("Writing client id to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
elif cnt == 2:
print("Writing client secret to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
line = fp.readline()
cnt += 1
client_path = 'app_clientcred.txt'
with open(client_path) as fp:
line = fp.readline()
cnt = 1
while line:
if cnt == 1:
print("Writing client id to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_id: ", value=line.rstrip())
elif cnt == 2:
print("Writing client secret to " + secrets_filepath)
modify_file(secrets_filepath, "uc_client_secret: ", value=line.rstrip())
line = fp.readline()
cnt += 1
def read_token_line(self):
token_path = 'app_usercred.txt'
with open(token_path) as fp:
line = fp.readline()
print("Writing access token to " + secrets_filepath)
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
token_path = 'app_usercred.txt'
with open(token_path) as fp:
line = fp.readline()
print("Writing access token to " + secrets_filepath)
modify_file(secrets_filepath, "uc_access_token: ", value=line.rstrip())
def read_config_line():
with open(config_filepath) as fp:
line = fp.readline()
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
with open(config_filepath) as fp:
line = fp.readline()
modify_file(config_filepath, "mastodon_hostname: ", value=hostname)
def log_in():
error = 0
try:
global hostname
hostname = input("Enter Mastodon hostname: ")
user_name = input("User name, ex. user@" + hostname +"? ")
user_password = getpass.getpass("User password? ")
app_name = input("This app name? ")
Mastodon.create_app(app_name, scopes=["read","write"],
to_file="app_clientcred.txt", api_base_url=hostname)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write"],
to_file = "app_usercred.txt"
)
except MastodonIllegalArgumentError as i_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(i_error)
except MastodonNetworkError as n_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(n_error)
except MastodonReadTimeout as r_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(r_error)
except MastodonAPIError as a_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(a_error)
finally:
if error == 0:
create_dir()
create_file()
write_params()
client_path = 'app_clientcred.txt'
read_client_lines(client_path)
token_path = 'app_usercred.txt'
read_token_line(token_path)
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt temp file..")
os.remove("app_clientcred.txt")
if os.path.exists("app_usercred.txt"):
print("Removing app_usercred.txt temp file..")
os.remove("app_usercred.txt")
print("Secrets setup done!\n")
error = 0
try:
global hostname
hostname = input("Enter Mastodon hostname: ")
user_name = input("User name, ex. user@" + hostname +"? ")
user_password = getpass.getpass("User password? ")
app_name = input("This app name? ")
Mastodon.create_app(app_name, scopes=["read","write"],
to_file="app_clientcred.txt", api_base_url=hostname)
mastodon = Mastodon(client_id = "app_clientcred.txt", api_base_url = hostname)
mastodon.log_in(
user_name,
user_password,
scopes = ["read", "write"],
to_file = "app_usercred.txt"
)
except MastodonIllegalArgumentError as i_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(i_error)
except MastodonNetworkError as n_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(n_error)
except MastodonReadTimeout as r_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(r_error)
except MastodonAPIError as a_error:
error = 1
if os.path.exists("secrets/secrets.txt"):
print("Removing secrets/secrets.txt file..")
os.remove("secrets/secrets.txt")
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt file..")
os.remove("app_clientcred.txt")
sys.exit(a_error)
finally:
if error == 0:
create_dir()
create_file()
write_params()
client_path = 'app_clientcred.txt'
read_client_lines(client_path)
token_path = 'app_usercred.txt'
read_token_line(token_path)
if os.path.exists("app_clientcred.txt"):
print("Removing app_clientcred.txt temp file..")
os.remove("app_clientcred.txt")
if os.path.exists("app_usercred.txt"):
print("Removing app_usercred.txt temp file..")
os.remove("app_usercred.txt")
print("Secrets setup done!\n")
def modify_file(file_name,pattern,value=""):
fh=fileinput.input(file_name,inplace=True)
@ -175,12 +174,17 @@ def get_hostname( parameter, config_filepath ):
print("hostname setup done!")
sys.exit(0)
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
###############################################################################
# main
# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath) # E.g., mastodon.social
if __name__ == '__main__':
# Load secrets from secrets file
secrets_filepath = "secrets/secrets.txt"
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
# Load configuration from config file
config_filepath = "config.txt"
mastodon_hostname = get_hostname("mastodon_hostname", config_filepath)