From b385833cbe1a05103390504faa99919de48fdafa Mon Sep 17 00:00:00 2001 From: cellarspoon Date: Wed, 15 Dec 2021 12:23:37 +0100 Subject: [PATCH] cli commands --- .gitignore | 1 + lumbunglib/cloudcal.py | 36 +++++++------- lumbunglib/feed.py | 101 ++++++++++++++++++--------------------- lumbunglib/hashtag.py | 105 +++++++++++++++++------------------------ lumbunglib/video.py | 97 +++++++++++++++++-------------------- pyproject.toml | 4 ++ 6 files changed, 154 insertions(+), 190 deletions(-) diff --git a/.gitignore b/.gitignore index 9daeafb..b1e61e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ test +__pycache__ diff --git a/lumbunglib/cloudcal.py b/lumbunglib/cloudcal.py index 98c546e..9b1d874 100644 --- a/lumbunglib/cloudcal.py +++ b/lumbunglib/cloudcal.py @@ -1,8 +1,3 @@ -#!/bin/python3 - -# lumbung.space calendar feed generator -# © 2021 roel roscam abbing gplv3 etc - import os import re import shutil @@ -187,22 +182,23 @@ def update_event_post(post_dir, event): print("Event current: ", event.name, "({})".format(event.uid)) -for event in list(cal.events): +def main(): + for event in list(cal.events): - post_dir = os.path.join(output_dir, event.uid) + post_dir = os.path.join(output_dir, event.uid) - if event.uid not in existing_posts: - # if there is an event we dont already have, make it - create_event_post(post_dir, event) + if event.uid not in existing_posts: + # if there is an event we dont already have, make it + create_event_post(post_dir, event) - elif event.uid in existing_posts: - # if we already have it, update - update_event_post(post_dir, event) - existing_posts.remove( - event.uid - ) # create list of posts which have not been returned by the calendar + elif event.uid in existing_posts: + # if we already have it, update + update_event_post(post_dir, event) + existing_posts.remove( + event.uid + ) # create list of posts which have not been returned by the calendar -for post in existing_posts: - # remove events not returned by the calendar (deletion) - print("deleted", post) - shutil.rmtree(os.path.join(output_dir, post)) + for post in existing_posts: + # remove events not returned by the calendar (deletion) + print("deleted", post) + shutil.rmtree(os.path.join(output_dir, post)) diff --git a/lumbunglib/feed.py b/lumbunglib/feed.py index 886e4a5..a68c550 100644 --- a/lumbunglib/feed.py +++ b/lumbunglib/feed.py @@ -1,8 +1,3 @@ -#!/bin/python3 - -# lumbung.space rss feed aggregator -# © 2021 roel roscam abbing gplv3 etc - import os import shutil import time @@ -100,6 +95,9 @@ def create_post(post_dir, entry): parsed_content = parse_posts(post_dir, post_content) + template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + template = env.get_template("feed.md") with open(os.path.join(post_dir, "index.html"), "w") as f: # n.b. .html post = template.render(frontmatter=frontmatter, content=parsed_content) f.write(post) @@ -140,7 +138,6 @@ def parse_posts(post_dir, post_content): """ soup = BeautifulSoup(post_content, "html.parser") allowed_iframe_sources = ["youtube.com", "vimeo.com", "tv.lumbung.space"] - media = [] for img in soup(["img", "object"]): local_image = grab_media(post_dir, img["src"]) @@ -184,70 +181,64 @@ def grab_feed(feed_url): return False -feed_urls = open("feeds_list.txt", "r").read().splitlines() +def main(): + feed_urls = open("feeds_list.txt", "r").read().splitlines() -start = time.time() + start = time.time() -if not os.path.exists("etags"): - os.mkdir("etags") + if not os.path.exists("etags"): + os.mkdir("etags") + output_dir = os.environ.get("OUTPUT_DIR") -template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") -env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) -output_dir = os.environ.get("OUTPUT_DIR") + # add iframe to the allowlist of feedparser's sanitizer, + # this is now handled in parse_post() + feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {"iframe"} -if not os.path.exists(output_dir): - os.makedirs(output_dir) + for feed_url in feed_urls: -template = env.get_template("feed.md") + feed_name = urlparse(feed_url).netloc -# add iframe to the allowlist of feedparser's sanitizer, -# this is now handled in parse_post() -feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {"iframe"} + feed_dir = os.path.join(output_dir, feed_name) -for feed_url in feed_urls: + if not os.path.exists(feed_dir): + os.makedirs(feed_dir) - feed_name = urlparse(feed_url).netloc + existing_posts = os.listdir(feed_dir) - feed_dir = os.path.join(output_dir, feed_name) + data = grab_feed(feed_url) - if not os.path.exists(feed_dir): - os.makedirs(feed_dir) + if data: + for entry in data.entries: + # if 'tags' in entry: + # for tag in entry.tags: + # for x in ['lumbung.space', 'D15', 'lumbung']: + # if x in tag['term']: + # print(entry.title) + entry["feed_name"] = feed_name - existing_posts = os.listdir(feed_dir) + post_name = slugify(entry.title) + post_dir = os.path.join(output_dir, feed_name, post_name) - data = grab_feed(feed_url) + if post_name not in existing_posts: + # if there is a blog entry we dont already have, make it + create_post(post_dir, entry) - if data: - for entry in data.entries: - # if 'tags' in entry: - # for tag in entry.tags: - # for x in ['lumbung.space', 'D15', 'lumbung']: - # if x in tag['term']: - # print(entry.title) - entry["feed_name"] = feed_name + elif post_name in existing_posts: + # if we already have it, update it + create_post(post_dir, entry) + existing_posts.remove( + post_name + ) # create list of posts which have not been returned by the feed - post_name = slugify(entry.title) - post_dir = os.path.join(output_dir, feed_name, post_name) + for post in existing_posts: + # remove blog posts no longer returned by the RSS feed + print("deleted", post) + shutil.rmtree(os.path.join(feed_dir, slugify(post))) - if post_name not in existing_posts: - # if there is a blog entry we dont already have, make it - create_post(post_dir, entry) + end = time.time() - elif post_name in existing_posts: - # if we already have it, update it - create_post(post_dir, entry) - existing_posts.remove( - post_name - ) # create list of posts which have not been returned by the feed - - for post in existing_posts: - # remove blog posts no longer returned by the RSS feed - print("deleted", post) - shutil.rmtree(os.path.join(feed_dir, slugify(post))) - - -end = time.time() - -print(end - start) + print(end - start) diff --git a/lumbunglib/hashtag.py b/lumbunglib/hashtag.py index 5111755..9e3f17c 100644 --- a/lumbunglib/hashtag.py +++ b/lumbunglib/hashtag.py @@ -1,9 +1,3 @@ -# lumbung.space hashtag publishing bot -# © 2021 roel roscam abbing agplv3 -# Makes Hugo posts out of hashtag feeds on Mastodon. -# Requires an account on the Mastodon instance configured. -# Currently does not do any thread recreation and only handles images - import os import shutil from pathlib import Path @@ -12,23 +6,11 @@ import jinja2 import requests from mastodon import Mastodon -# Which instance to login to instance = "https://social.lumbung.space" - -# n.b. if it is the first time you use this script -# You need to register the app: -# https://mastodonpy.readthedocs.io/en/stable/#module-mastodon - -# Login credentials for bot account email = "" password = "" - -# Which hashtags to publish hashtags = ["jalansesama"] -# your Hugo content directory -output_dir = os.environ.get("OUTPUT_DIR", "path/to/hugo/content") - def login_mastodon_bot(): mastodon = Mastodon( @@ -82,6 +64,14 @@ def create_post(post_directory, post_metadata): if not os.path.exists(post_directory): os.mkdir(post_directory) + template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + + env.filters["localize_media_url"] = localize_media_url + env.filters["filter_mastodon_urls"] = filter_mastodon_urls + + template = env.get_template("hashtag.md") + with open(os.path.join(post_directory, "index.html"), "w") as f: post = template.render(post_metadata=post_metadata) f.write(post) @@ -106,57 +96,48 @@ def filter_mastodon_urls(content): return content -mastodon = login_mastodon_bot() +def main(): + mastodon = login_mastodon_bot() -output_dir = output_dir + output_dir = os.environ.get("OUTPUT_DIR") + if not os.path.exists(output_dir): + os.mkdir(output_dir) + for hashtag in hashtags: -template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") -env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + hashtag_dir = os.path.join(output_dir, hashtag) + if not os.path.exists(hashtag_dir): + os.mkdir(hashtag_dir) -env.filters["localize_media_url"] = localize_media_url -env.filters["filter_mastodon_urls"] = filter_mastodon_urls + existing_posts = os.listdir(hashtag_dir) # list all existing posts -template = env.get_template("hashtag.md") + timeline = mastodon.timeline_hashtag( + hashtag, local=True, only_media=True + ) # returns max 20 queries and only with media + timeline = mastodon.fetch_remaining( + timeline + ) # returns all the rest n.b. can take a while because of rate limit + for post_metadata in timeline: + post_dir = os.path.join(hashtag_dir, str(post_metadata["id"])) -if not os.path.exists(output_dir): - os.mkdir(output_dir) + # if there is a post in the feed we dont already have locally, make it + if str(post_metadata["id"]) not in existing_posts: + if not post_metadata[ + "local_only" + ]: # if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature + create_post(post_dir, post_metadata) -for hashtag in hashtags: + # if we already have the post do nothing, possibly update + elif str(post_metadata["id"]) in existing_posts: + # update_post(post_dir, post_metadata) + existing_posts.remove( + str(post_metadata["id"]) + ) # create list of posts which have not been returned in the feed - hashtag_dir = os.path.join(output_dir, hashtag) - if not os.path.exists(hashtag_dir): - os.mkdir(hashtag_dir) - - existing_posts = os.listdir(hashtag_dir) # list all existing posts - - timeline = mastodon.timeline_hashtag( - hashtag, local=True, only_media=True - ) # returns max 20 queries and only with media - timeline = mastodon.fetch_remaining( - timeline - ) # returns all the rest n.b. can take a while because of rate limit - - for post_metadata in timeline: - post_dir = os.path.join(hashtag_dir, str(post_metadata["id"])) - - # if there is a post in the feed we dont already have locally, make it - if str(post_metadata["id"]) not in existing_posts: - - if not post_metadata[ - "local_only" - ]: # if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature - create_post(post_dir, post_metadata) - - # if we already have the post do nothing, possibly update - elif str(post_metadata["id"]) in existing_posts: - # update_post(post_dir, post_metadata) - existing_posts.remove( - str(post_metadata["id"]) - ) # create list of posts which have not been returned in the feed - - for post in existing_posts: - print("deleted", post) # rm posts that exist but are no longer returned in feed - shutil.rmtree(os.path.join(hashtag_dir, post)) + for post in existing_posts: + print( + "deleted", post + ) # rm posts that exist but are no longer returned in feed + shutil.rmtree(os.path.join(hashtag_dir, post)) diff --git a/lumbunglib/video.py b/lumbunglib/video.py index ecf1d9b..4194467 100644 --- a/lumbunglib/video.py +++ b/lumbunglib/video.py @@ -1,8 +1,3 @@ -#!/bin/python3 - -# lumbung.space video feed generator -# c 2021 roel roscam abbing gpvl3 etc - import ast import datetime import json @@ -15,6 +10,9 @@ import jinja2 import peertube import requests +host = "https://tv.lumbung.space" +configuration = peertube.Configuration(host=host + "/api/v1") +client = peertube.ApiClient(configuration) # jinja filters & config def duration(n): @@ -35,29 +33,10 @@ def linebreaks(text): return br.sub(r"
\n", text) -template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") -env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) -env.filters["duration"] = duration -env.filters["linebreaks"] = linebreaks - -host = "https://tv.lumbung.space" - -configuration = peertube.Configuration(host=host + "/api/v1") - -client = peertube.ApiClient(configuration) - -v = peertube.VideoApi(client) - -response = v.videos_get(count=100, filter="local", tags_one_of="publish") - -videos = response.to_dict() -videos = videos["data"] - - -def create_post(post_directory, video_metadata): +def create_post(post_directory, video_metadata, host): global client # lazy - if not os.path.exists(post_dir): + if not os.path.exists(post_directory): os.mkdir(post_directory) preview_image = video_metadata["preview_path"].split("/")[-1] @@ -77,6 +56,12 @@ def create_post(post_directory, video_metadata): long_description = ast.literal_eval(api_response) video_metadata["description"] = long_description["description"] + template_dir = os.path.join(Path(__file__).parent.resolve(), "templates") + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir)) + env.filters["duration"] = duration + env.filters["linebreaks"] = linebreaks + template = env.get_template("video.md") + with open(os.path.join(post_directory, "index.md"), "w") as f: post = template.render(v=video_metadata, host=host, preview_image=preview_image) f.write(post) @@ -86,7 +71,7 @@ def create_post(post_directory, video_metadata): f.write(timestamp.format("X")) -def update_post(post_directory, video_metadata): +def update_post(post_directory, video_metadata, host): if os.path.exists(post_directory): if os.path.exists(os.path.join(post_directory, ".timestamp")): old_timestamp = open(os.path.join(post_directory, ".timestamp")).read() @@ -101,7 +86,7 @@ def update_post(post_directory, video_metadata): video_metadata["name"], "({})".format(video_metadata["uuid"]), ) - create_post(post_dir, video_metadata) + create_post(post_directory, video_metadata, host) else: print( "Video current: ", @@ -110,37 +95,43 @@ def update_post(post_directory, video_metadata): ) else: # compat for when there is no timestamp yet.. - create_post(post_dir, video_metadata) + create_post(post_directory, video_metadata, host) -output_dir = os.environ.get( - "OUTPUT_DIR", "/home/r/Programming/lumbung.space/lumbung.space-web/content/video" -) +def main(): + v = peertube.VideoApi(client) -if not os.path.exists(output_dir): - os.mkdir(output_dir) + response = v.videos_get(count=100, filter="local", tags_one_of="publish") -template = env.get_template("video.md") + videos = response.to_dict() + videos = videos["data"] -existing_posts = os.listdir(output_dir) + output_dir = os.environ.get("OUTPUT_DIR") -for video_metadata in videos: - post_dir = os.path.join(output_dir, video_metadata["uuid"]) + if not os.path.exists(output_dir): + os.mkdir(output_dir) - if ( - video_metadata["uuid"] not in existing_posts - ): # if there is a video we dont already have, make it - print("New: ", video_metadata["name"], "({})".format(video_metadata["uuid"])) - create_post(post_dir, video_metadata) + existing_posts = os.listdir(output_dir) - elif ( - video_metadata["uuid"] in existing_posts - ): # if we already have the video do nothing, possibly update - update_post(post_dir, video_metadata) - existing_posts.remove( - video_metadata["uuid"] - ) # create list of posts which have not been returned by peertube + for video_metadata in videos: + post_dir = os.path.join(output_dir, video_metadata["uuid"]) -for post in existing_posts: - print("deleted", post) # rm posts not returned - shutil.rmtree(os.path.join(output_dir, post)) + if ( + video_metadata["uuid"] not in existing_posts + ): # if there is a video we dont already have, make it + print( + "New: ", video_metadata["name"], "({})".format(video_metadata["uuid"]) + ) + create_post(post_dir, video_metadata, host) + + elif ( + video_metadata["uuid"] in existing_posts + ): # if we already have the video do nothing, possibly update + update_post(post_dir, video_metadata, host) + existing_posts.remove( + video_metadata["uuid"] + ) # create list of posts which have not been returned by peertube + + for post in existing_posts: + print("deleted", post) # rm posts not returned + shutil.rmtree(os.path.join(output_dir, post)) diff --git a/pyproject.toml b/pyproject.toml index e54b84b..ec2dda6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,7 @@ peertube = {git = "https://framagit.org/framasoft/peertube/clients/python.git"} [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +cal= "lumbunglib.cloudcal:main" +vid = "lumbunglib.video:main"