392 líneas
9,7 KiB
Python
392 líneas
9,7 KiB
Python
import os
|
|
import feedparser
|
|
from bs4 import BeautifulSoup
|
|
from mastodon import Mastodon
|
|
import psycopg2
|
|
import sys
|
|
import time
|
|
import requests
|
|
import shutil
|
|
import tweepy
|
|
from tweepy import TweepyException
|
|
import logging
|
|
import filetype
|
|
import ffmpeg
|
|
import pdb
|
|
|
|
logger = logging.getLogger()
|
|
|
|
def get_toot(title):
|
|
|
|
soup = BeautifulSoup(title, 'html.parser')
|
|
|
|
toot_text = soup.get_text()
|
|
|
|
sub_str = 'http'
|
|
find_link = toot_text.find(sub_str)
|
|
if find_link != -1:
|
|
|
|
tuit_text = toot_text[:toot_text.index(sub_str)]
|
|
|
|
else:
|
|
|
|
tuit_text = toot_text
|
|
|
|
links_lst = ''
|
|
for links in soup.find_all('a'):
|
|
find_tag = links.get('href').find('/tags/')
|
|
if find_tag == -1:
|
|
links_lst += links.get('href')
|
|
|
|
if len(links_lst) > 0:
|
|
|
|
last_text = toot_text[len(tuit_text) + len(links_lst):]
|
|
|
|
else:
|
|
|
|
last_text = ''
|
|
|
|
tuit_text = f'{tuit_text} {links_lst} {last_text}'
|
|
|
|
return tuit_text
|
|
|
|
def compose_tweet(tuit_text, with_images, is_reply):
|
|
|
|
if len(tuit_text) > 280:
|
|
|
|
if with_images:
|
|
|
|
tuit_text1 = tuit_text[:250].rsplit(' ', 1)[0] + ' (1/2)'
|
|
tuit_text2 = tuit_text[int(len(tuit_text1)-6):] + ' (2/2)'
|
|
|
|
else:
|
|
|
|
tuit_text1 = tuit_text[:275].rsplit(' ', 1)[0] + ' (1/2)'
|
|
tuit_text2 = tuit_text[int(len(tuit_text1)-6):] + ' (2/2)'
|
|
|
|
try:
|
|
|
|
if with_images:
|
|
|
|
images_id_lst = []
|
|
|
|
i = 0
|
|
while i < len(images_list):
|
|
|
|
kind = filetype.guess('images/' + images_list[i])
|
|
if kind.mime == 'video/mp4':
|
|
|
|
probe = ffmpeg.probe('images/' + images_list[i])
|
|
duration = probe['streams'][0]['duration']
|
|
|
|
if float(duration) > 139:
|
|
|
|
print(f'video duration is too large: {duration}')
|
|
|
|
else:
|
|
|
|
media = api.media_upload('images/' + images_list[i], media_category='tweet_video')
|
|
|
|
if media.processing_info['state'] == 'succeeded':
|
|
|
|
images_id_lst.append(media.media_id)
|
|
|
|
else:
|
|
|
|
print(media.processing_info)
|
|
|
|
else:
|
|
|
|
media = api.media_upload('images/' + images_list[i])
|
|
|
|
images_id_lst.append(media.media_id)
|
|
|
|
i += 1
|
|
|
|
if len(tuit_text) > 280:
|
|
|
|
if is_reply:
|
|
|
|
first_tweet = api.update_status(status=tuit_text1, in_reply_to_status_id=tweet_id)
|
|
|
|
else:
|
|
|
|
first_tweet = api.update_status(status=tuit_text1)
|
|
|
|
tweet = api.update_status(status=tuit_text2, in_reply_to_status_id=first_tweet.id, media_ids=images_id_lst)
|
|
|
|
else:
|
|
|
|
if is_reply:
|
|
|
|
tweet = api.update_status(status=tuit_text, in_reply_to_status_id=tweet_id, media_ids=images_id_lst)
|
|
|
|
else:
|
|
|
|
tweet = api.update_status(status=tuit_text, media_ids=images_id_lst)
|
|
|
|
else:
|
|
|
|
if len(tuit_text) > 280:
|
|
|
|
if is_reply:
|
|
|
|
first_tweet = api.update_status(tuit_text1, in_reply_to_status_id=tweet_id)
|
|
|
|
else:
|
|
|
|
first_tweet = api.update_status(tuit_text1)
|
|
|
|
tweet = api.update_status(tuit_text2, in_reply_to_status_id=first_tweet.id)
|
|
|
|
else:
|
|
|
|
if is_reply:
|
|
|
|
tweet = api.update_status(status=tuit_text, in_reply_to_status_id=tweet_id)
|
|
|
|
else:
|
|
|
|
tweet = api.update_status(tuit_text)
|
|
|
|
return tweet
|
|
|
|
except TweepyException as err:
|
|
|
|
print('\n')
|
|
sys.exit(err)
|
|
|
|
def get_tweet_id(toot_id):
|
|
|
|
tweet_id = 0
|
|
|
|
try:
|
|
|
|
conn = None
|
|
|
|
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute('select tweet_id from id where toot_id=(%s)', (toot_id,))
|
|
|
|
row = cur.fetchone()
|
|
|
|
if row != None:
|
|
|
|
tweet_id = row[0]
|
|
|
|
cur.close()
|
|
|
|
return(tweet_id)
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
def write_image(image_url):
|
|
|
|
if not os.path.exists('images'):
|
|
os.makedirs('images')
|
|
filename = image_url.split("/") [-1]
|
|
r = requests.get(image_url, stream = True)
|
|
r.raw.decode_content = True
|
|
with open('images/' + filename, 'wb') as f:
|
|
shutil.copyfileobj(r.raw, f)
|
|
|
|
return filename
|
|
|
|
def create_api():
|
|
|
|
auth = tweepy.OAuthHandler(api_key, api_key_secret)
|
|
auth.set_access_token(access_token, access_token_secret)
|
|
api = tweepy.API(auth)
|
|
try:
|
|
api.verify_credentials()
|
|
logged_in = True
|
|
except Exception as e:
|
|
logger.error("Error creating API", exc_info=True)
|
|
raise e
|
|
logger.info("API created")
|
|
return (api, logged_in)
|
|
|
|
def mastodon():
|
|
|
|
# Load secrets from secrets file
|
|
secrets_filepath = "secrets/secrets.txt"
|
|
uc_client_id = get_parameter("uc_client_id", secrets_filepath)
|
|
uc_client_secret = get_parameter("uc_client_secret", secrets_filepath)
|
|
uc_access_token = get_parameter("uc_access_token", secrets_filepath)
|
|
|
|
# Load configuration from config file
|
|
config_filepath = "config/config.txt"
|
|
mastodon_hostname = get_parameter("mastodon_hostname", config_filepath)
|
|
|
|
# Initialise Mastodon API
|
|
mastodon = Mastodon(
|
|
client_id=uc_client_id,
|
|
client_secret=uc_client_secret,
|
|
access_token=uc_access_token,
|
|
api_base_url='https://' + mastodon_hostname,
|
|
)
|
|
|
|
# Initialise access headers
|
|
headers = {'Authorization': 'Bearer %s'%uc_access_token}
|
|
|
|
return (mastodon, mastodon_hostname)
|
|
|
|
def db_config():
|
|
|
|
# Load db configuration from config file
|
|
db_config_filepath = "config/db_config.txt"
|
|
feeds_db = get_parameter("feeds_db", db_config_filepath)
|
|
feeds_db_user = get_parameter("feeds_db_user", db_config_filepath)
|
|
feeds_url = get_parameter("feeds_url", db_config_filepath)
|
|
|
|
return (feeds_db, feeds_db_user, feeds_url)
|
|
|
|
def twitter_config():
|
|
|
|
twitter_config_filepath = "config/keys_config.txt"
|
|
api_key = get_parameter("api_key", twitter_config_filepath)
|
|
api_key_secret = get_parameter("api_key_secret", twitter_config_filepath)
|
|
access_token = get_parameter("access_token", twitter_config_filepath)
|
|
access_token_secret = get_parameter("access_token_secret", twitter_config_filepath)
|
|
|
|
return(api_key, api_key_secret, access_token, access_token_secret)
|
|
|
|
# Returns the parameter from the specified file
|
|
def get_parameter( parameter, file_path ):
|
|
# Check if secrets file exists
|
|
if not os.path.isfile(file_path):
|
|
print("File %s not found, exiting."%file_path)
|
|
sys.exit(0)
|
|
|
|
# Find parameter in file
|
|
with open( file_path ) as f:
|
|
for line in f:
|
|
if line.startswith( parameter ):
|
|
return line.replace(parameter + ":", "").strip()
|
|
|
|
# Cannot find parameter, exit
|
|
print(file_path + " Missing parameter %s "%parameter)
|
|
sys.exit(0)
|
|
|
|
###############################################################################
|
|
# main
|
|
|
|
if __name__ == '__main__':
|
|
|
|
mastodon, mastodon_hostname = mastodon()
|
|
|
|
feeds_db, feeds_db_user, feeds_url = db_config()
|
|
|
|
api_key, api_key_secret, access_token, access_token_secret = twitter_config()
|
|
|
|
logged_in = False
|
|
|
|
try:
|
|
|
|
newsfeeds = feedparser.parse(feeds_url)
|
|
|
|
except:
|
|
|
|
print(newsfeeds.status)
|
|
sys.exit(0)
|
|
|
|
for entry in newsfeeds.entries:
|
|
|
|
publish = False
|
|
with_images = False
|
|
is_reply = False
|
|
|
|
title = entry['summary']
|
|
id = entry['id']
|
|
link = entry['link']
|
|
|
|
toot_id = link.rsplit('/')[4]
|
|
|
|
tweet_id = get_tweet_id(toot_id)
|
|
|
|
if tweet_id == 0:
|
|
|
|
publish = True
|
|
|
|
reply_id = mastodon.status(toot_id).in_reply_to_id
|
|
if reply_id != None:
|
|
|
|
is_reply = True
|
|
tweet_id = get_tweet_id(reply_id)
|
|
|
|
if len(entry.links) >= 2:
|
|
|
|
with_images = True
|
|
images_list = []
|
|
images = len(entry.links) - 1
|
|
|
|
i = 0
|
|
while i < images:
|
|
|
|
image_url = entry.links[i+1].href
|
|
image_filename = write_image(image_url)
|
|
images_list.append(image_filename)
|
|
i += 1
|
|
|
|
###########################################################
|
|
|
|
if publish:
|
|
|
|
tuit_text = get_toot(title)
|
|
|
|
print("Tooting...")
|
|
print(tuit_text)
|
|
|
|
if not logged_in:
|
|
|
|
api, logged_in = create_api()
|
|
|
|
tweet = compose_tweet(tuit_text, with_images, is_reply)
|
|
|
|
#########################################################
|
|
|
|
sql_insert_ids = 'INSERT INTO id(toot_id, tweet_id) VALUES (%s,%s)'
|
|
|
|
conn = None
|
|
|
|
try:
|
|
|
|
conn = psycopg2.connect(database = feeds_db, user = feeds_db_user, password = "", host = "/var/run/postgresql", port = "5432")
|
|
|
|
cur = conn.cursor()
|
|
|
|
cur.execute(sql_insert_ids, (toot_id, tweet.id))
|
|
|
|
conn.commit()
|
|
|
|
cur.close()
|
|
|
|
except (Exception, psycopg2.DatabaseError) as error:
|
|
|
|
print(error)
|
|
|
|
finally:
|
|
|
|
if conn is not None:
|
|
|
|
conn.close()
|
|
|
|
#########################################################
|
|
|
|
time.sleep(2)
|
|
|
|
else:
|
|
|
|
print("Any new feeds")
|
|
sys.exit(0)
|