318 líneas
8,6 KiB
Python
318 líneas
8,6 KiB
Python
import os
|
|
import feedparser
|
|
from bs4 import BeautifulSoup
|
|
from mastodon import Mastodon
|
|
import psycopg2
|
|
import sys
|
|
import time
|
|
import requests
|
|
import shutil
|
|
import tweepy
|
|
from tweepy import TweepError
|
|
import logging
|
|
import pdb
|
|
|
|
logger = logging.getLogger()
|
|
|
|
def write_image(image_url):
|
|
|
|
if not os.path.exists('images'):
|
|
os.makedirs('images')
|
|
filename = image_url.split("/") [-1]
|
|
r = requests.get(image_url, stream = True)
|
|
r.raw.decode_content = True
|
|
with open('images/' + filename, 'wb') as f:
|
|
shutil.copyfileobj(r.raw, f)
|
|
|
|
return filename
|
|
|
|
def create_api():
|
|
|
|
auth = tweepy.OAuthHandler(api_key, api_key_secret)
|
|
auth.set_access_token(access_token, access_token_secret)
|
|
api = tweepy.API(auth, wait_on_rate_limit=True,
|
|
wait_on_rate_limit_notify=True)
|
|
try:
|
|
api.verify_credentials()
|
|
logged_in = True
|
|
except Exception as e:
|
|
logger.error("Error creating API", exc_info=True)
|
|
raise e
|
|
logger.info("API created")
|
|
return (api, logged_in)
|
|
|
|
def mastodon():
|
|
|
|
# Load secrets from secrets file
|
|
secrets_filepath = "secrets/secrets.txt"
|
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
|
|
# Load configuration from config file
|
|
config_filepath = "config/config.txt"
|
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
|
|
# Initialise Mastodon API
|
|
mastodon = Mastodon(
|
|
client_id=uc_client_id,
|
|
client_secret=uc_client_secret,
|
|
access_token=uc_access_token,
|
|
api_base_url='https://' + mastodon_hostname,
|
|
)
|
|
|
|
# Initialise access headers
|
|
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
|
|
|
return (mastodon, mastodon_hostname)
|
|
|
|
def db_config():
|
|
|
|
# Load db configuration from config file
|
|
db_config_filepath = "config/db_config.txt"
|
|
feeds_db = get_parameter("feeds_db", db_config_filepath)
|
|
feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
|
|
feeds_url = get_parameter("feeds_url", db_config_filepath)
|
|
|
|
return (feeds_db, feeds_db_user, feeds_url)
|
|
|
|
def twitter_config():
|
|
|
|
twitter_config_filepath = "config/keys_config.txt"
|
|
api_key = get_parameter("api_key", twitter_config_filepath)
|
|
api_key_secret = get_parameter("api_key_secret", twitter_config_filepath)
|
|
access_token = get_parameter("access_token", twitter_config_filepath)
|
|
access_token_secret = get_parameter("access_token_secret", twitter_config_filepath)
|
|
|
|
return(api_key, api_key_secret, access_token, access_token_secret)
|
|
|
|
# Returns the parameter from the specified file
|
|
def get_parameter( parameter, file_path ):
|
|
# Check if secrets file exists
|
|
if not os.path.isfile(file_path):
|
|
print("File %s not found, exiting."%file_path)
|
|
sys.exit(0)
|
|
|
|
# Find parameter in file
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
# Cannot find parameter, exit
|
|
print(file_path + " Missing parameter %s "%parameter)
|
|
sys.exit(0)
|
|
|
|
###############################################################################
|
|
# main
|
|
|
|
if __name__ == '__main__':
|
|
|
|
#######################################################################
|
|
#mastodon, mastodon_hostname = mastodon()
|
|
|
|
feeds_db, feeds_db_user, feeds_url = db_config()
|
|
|
|
api_key, api_key_secret, access_token, access_token_secret = twitter_config()
|
|
|
|
logged_in = False
|
|
|
|
try:
|
|
|
|
newsfeeds = feedparser.parse(feeds_url)
|
|
|
|
except:
|
|
|
|
print(newsfeeds.status)
|
|
sys.exit(0)
|
|
|
|
for entry in newsfeeds.entries:
|
|
|
|
publish = False
|
|
with_images = False
|
|
|
|
title = entry['summary']
|
|
id = entry['id']
|
|
link = entry['link']
|
|
|
|
if len(entry.links) >= 2:
|
|
|
|
with_images = True
|
|
images_list = []
|
|
images = len(entry.links) - 1
|
|
|
|
i = 0
|
|
while i < images:
|
|
|
|
image_url = entry.links[i+1].href
|
|
image_filename = write_image(image_url)
|
|
images_list.append(image_filename)
|
|
i += 1
|
|
|
|
###################################################################
|
|
# check database if feed is already published
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('select link from feeds where link=(%s)', (link,))
|
|
|
|
row = cur.fetchone()
|
|
if row == None:
|
|
publish = True
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
###########################################################
|
|
|
|
if publish:
|
|
|
|
soup = BeautifulSoup(title, 'html.parser')
|
|
|
|
toot_text = soup.get_text()
|
|
|
|
sub_str = 'http'
|
|
find_link = toot_text.find(sub_str)
|
|
if find_link != -1:
|
|
|
|
tuit_text = toot_text[:toot_text.index(sub_str)]
|
|
|
|
else:
|
|
|
|
tuit_text = toot_text
|
|
|
|
links_lst = ''
|
|
for links in soup.find_all('a'):
|
|
find_tag = links.get('href').find('/tags/')
|
|
if find_tag == -1:
|
|
links_lst += links.get('href')
|
|
|
|
if len(links_lst) > 0:
|
|
|
|
last_text = toot_text[len(tuit_text) + len(links_lst):]
|
|
|
|
else:
|
|
|
|
last_text = ''
|
|
|
|
tuit_text = f'{tuit_text} {links_lst} {last_text}'
|
|
|
|
print("Tooting...")
|
|
print(tuit_text)
|
|
|
|
if not logged_in:
|
|
|
|
api, logged_in = create_api()
|
|
|
|
if len(tuit_text) < 280:
|
|
|
|
try:
|
|
|
|
if with_images:
|
|
|
|
images_id_lst = []
|
|
|
|
i = 0
|
|
while i < len(images_list):
|
|
|
|
media = api.media_upload('images/' + images_list[i])
|
|
images_id_lst.append(media.media_id)
|
|
i += 1
|
|
|
|
api.update_status(status=tuit_text, media_ids=images_id_lst)
|
|
|
|
else:
|
|
|
|
api.update_status(tuit_text)
|
|
|
|
except TweepError as err:
|
|
|
|
print('\n')
|
|
sys.exit(err)
|
|
|
|
else:
|
|
|
|
if with_images:
|
|
|
|
tuit_text1 = tuit_text[:250].rsplit(' ', 1)[0] + ' (1/2)'
|
|
tuit_text2 = tuit_text[int(len(tuit_text1)-6):] + ' (2/2)'
|
|
|
|
else:
|
|
|
|
tuit_text1 = tuit_text[:275].rsplit(' ', 1)[0] + ' (1/2)'
|
|
tuit_text2 = tuit_text[int(len(tuit_text1)-6):] + ' (2/2)'
|
|
|
|
try:
|
|
|
|
if with_images:
|
|
|
|
images_id_lst = []
|
|
|
|
i = 0
|
|
while i < len(images_list):
|
|
|
|
media = api.media_upload('images/' + images_list[i])
|
|
images_id_lst.append(media.media_id)
|
|
i += 1
|
|
|
|
first_tweet = api.update_status(status=tuit_text1)
|
|
api.update_status(status=tuit_text2, in_reply_to_status_id=first_tweet.id, media_ids=images_id_lst)
|
|
|
|
else:
|
|
|
|
first_tweet = api.update_status(tuit_text1)
|
|
api.update_status(tuit_text2, in_reply_to_status_id=first_tweet.id)
|
|
|
|
except TweepError as err:
|
|
|
|
print('\n')
|
|
sys.exit(err)
|
|
|
|
time.sleep(2)
|
|
|
|
#########################################################
|
|
|
|
insert_line = 'INSERT INTO feeds(link) VALUES (%s)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(insert_line, (link,))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
else:
|
|
|
|
print("Any new feeds")
|
|
sys.exit(0)
|