only return an author if there is one

Reviewed-on: ruangrupa/lumbunglib#33

lumbunglib/feed.py

@@ -5,6 +5,7 @@ from hashlib import md5
 from ast import literal_eval as make_tuple
 from pathlib import Path
 from urllib.parse import urlparse
+from re import sub
 
 import arrow
 import feedparser
@@ -84,6 +85,10 @@ def create_frontmatter(entry):
         for t in entry.tags:
             tags.append(t['term'])
 
+    card_type = "network"
+    if entry.feed_name == "pen.lumbung.space":
+        card_type = "pen"
+
     if "opds" in entry:
         frontmatter = {
         'title':entry.title,
@@ -104,7 +109,8 @@ def create_frontmatter(entry):
         'author': author,
         'original_link': entry.link,
         'feed_name': entry['feed_name'],
-        'tags': str(tags)
+        'tags': str(tags),
+        'card_type': card_type
         }
 
     return frontmatter
@@ -195,9 +201,10 @@ def parse_posts(post_dir, post_content):
     allowed_iframe_sources = ["youtube.com", "vimeo.com", "tv.lumbung.space"]
 
     for img in soup(["img", "object"]):
-        local_image = grab_media(post_dir, img["src"])
-        if img["src"] != local_image:
-            img["src"] = local_image
+        if img.get("src") != None:
+            local_image = grab_media(post_dir, img["src"])
+            if img["src"] != local_image:
+                img["src"] = local_image
 
     for iframe in soup(["iframe"]):
         if not any(source in iframe["src"] for source in allowed_iframe_sources):

lumbunglib/hashtag.py

@@ -19,13 +19,16 @@ hashtags = [
     "majelisakbar",
     "warungkopi",
     "lumbungkios",
+    "kassel_ecosystem",
+    "ruruhaus",
+    "offbeatentrack_kassel",
+    "lumbungofpublishers",
 ]
 
 
 def login_mastodon_bot():
     mastodon = Mastodon(
-        access_token=os.environ.get("MASTODON_AUTH_TOKEN"),
-        api_base_url = instance
+        access_token=os.environ.get("MASTODON_AUTH_TOKEN"), api_base_url=instance
     )
 
     return mastodon
@@ -69,9 +72,9 @@ def create_post(post_directory, post_metadata):
 
     template_dir = os.path.join(Path(__file__).parent.resolve(), "templates")
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
-    name = post_metadata['account']['display_name']
+    name = post_metadata["account"]["display_name"]
     name = sub('"', '\\"', name)
-    post_metadata['account']['display_name'] = name
+    post_metadata["account"]["display_name"] = name
     env.filters["localize_media_url"] = localize_media_url
     env.filters["filter_mastodon_urls"] = filter_mastodon_urls
 
@@ -136,7 +139,10 @@ def main():
                     create_post(post_dir, post_metadata)
                     all_existing_posts.append(str(post_metadata["id"]))
                 else:
-                    print("not pulling post %s (post is local only)" % (post_metadata["id"]))
+                    print(
+                        "not pulling post %s (post is local only)"
+                        % (post_metadata["id"])
+                    )
 
             # if we already have the post do nothing, possibly update
             elif str(post_metadata["id"]) in existing_posts:
@@ -145,7 +151,10 @@ def main():
                     str(post_metadata["id"])
                 )  # create list of posts which have not been returned in the feed
             elif str(post_metadata["id"]) in all_existing_posts:
-                print("skipping post %s as it was already pulled with a different hashtag." % (str(post_metadata["id"])))
+                print(
+                    "skipping post %s as it was already pulled with a different hashtag."
+                    % (str(post_metadata["id"]))
+                )
 
         for post in existing_posts:
             print(

lumbunglib/templates/feed.md

@@ -3,10 +3,10 @@ title: "{{ frontmatter.title }}"
 date: "{{ frontmatter.date }}" #2021-06-10T10:46:33+02:00
 draft: false
 summary: "{{ frontmatter.summary }}"
-author: "{{ frontmatter.author }}"
+authors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
-categories: ["network", "{{ frontmatter.feed_name}}"]
+categories: ["{{ frontmatter.card_type }}", "{{ frontmatter.feed_name}}"]
 tags: {{ frontmatter.tags }}
 ---
 

lumbunglib/templates/hashtag.md

@@ -1,12 +1,12 @@
 ---
-date: "{{ post_metadata.created_at }}" #2021-06-10T10:46:33+02:00
+date: {{ post_metadata.created_at }} #2021-06-10T10:46:33+02:00
 draft: false
-author: "{{ post_metadata.account.display_name }}"
-avatar: "{{ post_metadata.account.avatar }}"
+authors: ["{{ post_metadata.account.display_name }}"]
+avatar: {{ post_metadata.account.avatar }}
 categories: ["shouts"]
-images: [{% for i in post_metadata.media_attachments %} "{{ i.url }}", {% endfor %}]
-title: "{{ post_metadata.account.display_name }}"
-tags: [{% for i in post_metadata.tags %} "{{ i.name }}", {% endfor %}]
+images: [{% for i in post_metadata.media_attachments %} {{ i.url }}, {% endfor %}]
+title: {{ post_metadata.account.display_name }}
+tags: [{% for i in post_metadata.tags %} {{ i.name }} {% endfor %}]
 ---
 
 {% for item in post_metadata.media_attachments %}

lumbunglib/video.py

@@ -102,52 +102,60 @@ def main():
     v = peertube.VideoApi(client)
     count = 100
     page = 0
-    response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
-
-    videos = response.to_dict()
-    total = videos['total']
-    videos = videos['data']
-    total -= count
-    if total > 0:
-        to_download = total // count
-        last_page = total % count
-        for i in range(to_download):
-            page += 1
-            response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
-            videos += response.to_dict()['data']
-        if last_page > 0:
-            page += 1
-            response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
-            videos += response.to_dict()['data'][-1*last_page:]
+    try:
+        response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
+        videos = response.to_dict()
+        total = videos['total']
+        videos = videos['data']
+        total -= count
+        if total > 0:
+            to_download = total // count
+            last_page = total % count
+            for i in range(to_download):
+                page += 1
+                response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
+                videos += response.to_dict()['data']
+            if last_page > 0:
+                page += 1
+                response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
+                videos += response.to_dict()['data'][-1*last_page:]
 
 
-    output_dir = os.environ.get("OUTPUT_DIR")
+        output_dir = os.environ.get("OUTPUT_DIR")
 
-    if not os.path.exists(output_dir):
-        os.mkdir(output_dir)
+        if not os.path.exists(output_dir):
+            os.mkdir(output_dir)
 
-    existing_posts = os.listdir(output_dir)
+        existing_posts = os.listdir(output_dir)
 
-    for video_metadata in videos:
-        post_name = slugify(video_metadata["name"]) + "-" + video_metadata["uuid"]
-        post_dir = os.path.join(output_dir, post_name)
+        for video_metadata in videos:
+            post_name = slugify(video_metadata["name"]) + "-" + video_metadata["uuid"]
+            post_dir = os.path.join(output_dir, post_name)
 
-        if (
-            post_name not in existing_posts
-        ):  # if there is a video we dont already have, make it
-            print(
-                "New: ", video_metadata["name"], "({})".format(video_metadata["uuid"])
-            )
-            create_post(post_dir, video_metadata, host)
+            if (
+                post_name not in existing_posts
+            ):  # if there is a video we dont already have, make it
+                print(
+                    "New: ", video_metadata["name"], "({})".format(video_metadata["uuid"])
+                )
+                create_post(post_dir, video_metadata, host)
 
-        elif (
-            post_name in existing_posts
-        ):  # if we already have the video do nothing, possibly update
-            update_post(post_dir, video_metadata, host)
-            existing_posts.remove(
-                post_name
-            )  # create list of posts which have not been returned by peertube
+            elif (
+                post_name in existing_posts
+            ):  # if we already have the video do nothing, possibly update
+                update_post(post_dir, video_metadata, host)
+                existing_posts.remove(
+                    post_name
+                )  # create list of posts which have not been returned by peertube
+
+    except:
+        print("didn't get a response from peertube, instance might have been taken down or made private. removing all posts.")
+        output_dir = os.environ.get("OUTPUT_DIR")
+        if not os.path.exists(output_dir):
+            os.mkdir(output_dir)
+        existing_posts = os.listdir(output_dir)
 
     for post in existing_posts:
         print("deleted", post)  # rm posts not returned
         shutil.rmtree(os.path.join(output_dir, post))
+