ruangrupa/konfluks
2
0
Fork 2
Code Issues 9 Pull Requests Projects Releases Wiki Activity

deps and autoformat

Browse Source
This commit is contained in:
cellarspoon
2021-12-15 11:41:35 +01:00
parent 95964862b7
commit be3e14ac6c
16 changed files with 610 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
lumbung-hashtag-bot/README.md
View File

@ -1,30 +0,0 @@
# lumbung.space hashtag publishing bot
This script makes [Hugo page bundles](https://gohugo.io/content-management/page-bundles/) out of Hashtag feeds on a Mastodon Hometown or Glitchsoc instance.
## Install requirements
`pip3 install Mastodon.py jinja2`
## Setup
This script requires access to an account on said Mastodon instance. This instance and the credentials can be set in `config_hashtag_bot.py`.
If it is the first time you are running the script, you need to register the application on the Mastodon instance. Have a look at the [Mastodon.py documentation](https://mastodonpy.readthedocs.io/en/stable/#module-mastodon) for how to do that.
This bot only uses read permissions.
Set which hashtags you want to publish by adding them to the list `hashtags` in `config_hashtag_bot.py`. Omit the '#'.
## What it does
* The Bot only looks at the **local timeline** for posts under each hashtag configured in `config_hashtag_bot.py`.
* This means posts need to be **public** or directly addressed to the bot
* This script respects the mental model of 'local only' posts in the sense that people do not expect them to appear elsewhere. So **local only posts are ignored**
* It takes only posts with Media attached and then only those with images
## What it doesn't do
* Different types of media or embeds
* No thread recreation, each post is treated as a top level post

2
lumbung-hashtag-bot/post_template.md
View File

@ -11,4 +11,4 @@ tags: [{% for i in post_metadata.tags %} "{{ i.name }}", {% endfor %}]
<img src="{{item.url | localize_media_url }}" alt="{{item.description}}">
{% endfor %}
{{ post_metadata.content | filter_mastodon_urls }}
{{ post_metadata.content | filter_mastodon_urls }}

86
lumbung-hashtag-bot/publish_hashtags.py
View File

@ -5,28 +5,31 @@
# Currently does not do any thread recreation and only handles images
import os
import requests
import shutil
import jinja2
import requests
from mastodon import Mastodon
import config_hashtag_bot
def login_mastodon_bot():
mastodon = Mastodon(
client_id = 'publishbot_clientcred.secret',
api_base_url = config_hashtag_bot.instance
client_id="publishbot_clientcred.secret",
api_base_url=config_hashtag_bot.instance,
)
mastodon.log_in(
config_hashtag_bot.email,
config_hashtag_bot.password,
to_file = 'publishbot_usercred.secret', scopes=['read']
to_file="publishbot_usercred.secret",
scopes=["read"],
)
return mastodon
def create_frontmatter(post_metadata):
"""
Parse post metadata and return it as HUGO frontmatter
@ -35,6 +38,7 @@ def create_frontmatter(post_metadata):
frontmatter = ""
return frontmatter
def download_media(post_directory, media_attachments):
"""
Download media attached to posts. N.b. currently only images
@ -42,15 +46,16 @@ def download_media(post_directory, media_attachments):
"""
for item in media_attachments:
if item['type'] == 'image':
image = localize_media_url(item['url'])
#TODO check whether this needs to handle delete & redraft with different images
if item["type"] == "image":
image = localize_media_url(item["url"])
# TODO check whether this needs to handle delete & redraft with different images
if not os.path.exists(os.path.join(post_directory, image)):
#download image
response = requests.get(item['url'], stream=True)
with open(os.path.join(post_directory, image), 'wb') as img_file:
# download image
response = requests.get(item["url"], stream=True)
with open(os.path.join(post_directory, image), "wb") as img_file:
shutil.copyfileobj(response.raw, img_file)
print('Downloaded cover image', image)
print("Downloaded cover image", image)
def create_post(post_directory, post_metadata):
"""
@ -61,17 +66,18 @@ def create_post(post_directory, post_metadata):
if not os.path.exists(post_directory):
os.mkdir(post_directory)
with open(os.path.join(post_directory,'index.html'),'w') as f:
with open(os.path.join(post_directory, "index.html"), "w") as f:
post = template.render(post_metadata=post_metadata)
f.write(post)
download_media(post_directory, post_metadata['media_attachments'])
download_media(post_directory, post_metadata["media_attachments"])
def localize_media_url(url):
"""
Returns the filename, used also as custom jinja filter
"""
return url.split('/')[-1]
return url.split("/")[-1]
def filter_mastodon_urls(content):
@ -80,7 +86,7 @@ def filter_mastodon_urls(content):
e.g. <a href="https://social.lumbung.space/tags/jalankita" class="mention hashtag" rel="tag">
Used also as custom jinja filter
"""
#TODO
# TODO
return content
@ -89,16 +95,13 @@ mastodon = login_mastodon_bot()
output_dir = config_hashtag_bot.output_dir
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.curdir)
)
env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.curdir))
env.filters['localize_media_url'] = localize_media_url
env.filters['filter_mastodon_urls'] = filter_mastodon_urls
env.filters["localize_media_url"] = localize_media_url
env.filters["filter_mastodon_urls"] = filter_mastodon_urls
template = env.get_template('post_template.md')
template = env.get_template("post_template.md")
if not os.path.exists(output_dir):
os.mkdir(output_dir)
@ -110,28 +113,33 @@ for hashtag in config_hashtag_bot.hashtags:
if not os.path.exists(hashtag_dir):
os.mkdir(hashtag_dir)
existing_posts = os.listdir(hashtag_dir) #list all existing posts
existing_posts = os.listdir(hashtag_dir) # list all existing posts
timeline = mastodon.timeline_hashtag(hashtag, local=True, only_media=True) #returns max 20 queries and only with media
timeline = mastodon.fetch_remaining(timeline) #returns all the rest n.b. can take a while because of rate limit
timeline = mastodon.timeline_hashtag(
hashtag, local=True, only_media=True
) # returns max 20 queries and only with media
timeline = mastodon.fetch_remaining(
timeline
) # returns all the rest n.b. can take a while because of rate limit
for post_metadata in timeline:
post_dir = os.path.join(hashtag_dir, str(post_metadata['id']))
post_dir = os.path.join(hashtag_dir, str(post_metadata["id"]))
#if there is a post in the feed we dont already have locally, make it
if str(post_metadata['id']) not in existing_posts:
# if there is a post in the feed we dont already have locally, make it
if str(post_metadata["id"]) not in existing_posts:
if not post_metadata['local_only']: #if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature
if not post_metadata[
"local_only"
]: # if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature
create_post(post_dir, post_metadata)
# if we already have the post do nothing, possibly update
elif str(post_metadata['id']) in existing_posts:
#update_post(post_dir, post_metadata)
existing_posts.remove(str(post_metadata['id'])) # create list of posts which have not been returned in the feed
elif str(post_metadata["id"]) in existing_posts:
# update_post(post_dir, post_metadata)
existing_posts.remove(
str(post_metadata["id"])
) # create list of posts which have not been returned in the feed
for post in existing_posts:
print('deleted', post) #rm posts that exist but are no longer returned in feed
shutil.rmtree(os.path.join(hashtag_dir,post))
print("deleted", post) # rm posts that exist but are no longer returned in feed
shutil.rmtree(os.path.join(hashtag_dir, post))