do proper deletion

README.md

@@ -70,7 +70,7 @@ poetry install
 
 Each script requires some environment variables to run, you can see the latest deployment configuration over [here](https://git.autonomic.zone/ruangrupa/lumbung.space/src/branch/main/compose.yml), look for the values under the `environment: ...` stanza.
 
-All scripts have an entrypoint described in the [`pypoetry.toml`](https://git.autonomic.zone/ruangrupa/konfluks/src/commit/40bf9416b8792c08683ad8ac878093c7ef1b2f5d/pyproject.toml#L27-L31) which you can run via `poetry run ...`. For example, if you want to run the [`konfluks/video.py`](./knofluks/video.py) script, you'd do:
+All scripts have an entrypoint described in the [`pypoetry.toml`](./pyproject.toml) which you can run via `poetry run ...`. For example, if you want to run the [`konfluks/video.py`](./konfluks/video.py) script, you'd do:
 
 ```
 mkdir -p testdir

konfluks/calendars.py

@@ -138,9 +138,9 @@ def create_event_post(post_dir, event):
     for img in event_metadata["images"]:
 
         # parse img url to safe local image name
-        img_name = img.split("/")[-1]
+        img_name = os.path.basename(img)
-        fn, ext = img_name.split(".")
+        fn, ext = os.path.splitext(img_name)
-        img_name = slugify(fn) + "." + ext
+        img_name =  slugify(fn) + '.' + ext
 
         local_image = os.path.join(post_dir, img_name)
 

konfluks/feed.py

@@ -156,7 +156,8 @@ def parse_enclosures(post_dir, entry):
             print("found enclosed media", e.type)
             if "image/" in e.type:
                 featured_image = grab_media(post_dir, e.href)
-                entry["featured_image"] = featured_image
+                media_item = urlparse(e.href).path.split('/')[-1]
+                entry["featured_image"] = media_item
             else:
                 print("FIXME:ignoring enclosed", e.type)
     return entry
@@ -373,16 +374,16 @@ def main():
 
         data = grab_feed(feed_url)
 
-        if data:
+        if data: #whenever we get a 200
-
+            if data.feed: #only if it is an actual feed
                 opds_feed = False
+                if 'links' in data.feed:
                     for i in data.feed['links']:
                             if i['rel'] == 'self':
                                 if 'opds' in i['type']:
                                     opds_feed = True
                                     print("OPDS type feed!")
 
-
                 for entry in data.entries:
                     # if 'tags' in entry:
                     #     for tag in entry.tags:
@@ -425,10 +426,14 @@ def main():
                             post_name
                         )  # create list of posts which have not been returned by the feed
 
+
                 for post in existing_posts:
                     # remove blog posts no longer returned by the RSS feed
-                print("deleted", post)
+                    post_dir = os.path.join(output_dir, feed_name, post)
-                shutil.rmtree(os.path.join(feed_dir, slugify(post)))
+                    shutil.rmtree(post_dir)
+                    print("deleted", post_dir)
+            else:
+                print(feed_url, "is not or no longer a feed!")
 
     end = time.time()
 

konfluks/hashtag.py

@@ -60,6 +60,21 @@ def download_media(post_directory, media_attachments):
                 with open(os.path.join(post_directory, image), "wb") as img_file:
                     shutil.copyfileobj(response.raw, img_file)
                     print("Downloaded cover image", image)
+        elif item["type"] == "video":
+            video = localize_media_url(item["url"])
+            if not os.path.exists(os.path.join(post_directory, video)):
+                # download video file
+                response = requests.get(item["url"], stream=True)
+                with open(os.path.join(post_directory, video), "wb") as video_file:
+                    shutil.copyfileobj(response.raw, video_file)
+                    print("Downloaded video in post", video)
+            if not os.path.exists(os.path.join(post_directory, "thumbnail.png")):
+                #download video preview
+                response = requests.get(item["preview_url"], stream=True)
+                with open(os.path.join(post_directory, "thumbnail.png"), "wb") as thumbnail:
+                    shutil.copyfileobj(response.raw, thumbnail)
+                    print("Downloaded thumbnail for", video)
+
 
 
 def create_post(post_directory, post_metadata):
@@ -78,7 +93,6 @@ def create_post(post_directory, post_metadata):
     post_metadata["account"]["display_name"] = name
     env.filters["localize_media_url"] = localize_media_url
     env.filters["filter_mastodon_urls"] = filter_mastodon_urls
-
     template = env.get_template("hashtag.md")
 
     with open(os.path.join(post_directory, "index.html"), "w") as f:

konfluks/templates/calendar.md

@@ -2,7 +2,7 @@
 title: "{{ event.name }}"
 date: "{{ event.begin }}" #2021-06-10T10:46:33+02:00
 draft: false
-categories: "calendar"
+source: "lumbung calendar"
 event_begin: "{{ event.begin }}"
 event_end: "{{ event.end }}"
 duration: "{{ event.duration }}"

konfluks/templates/feed.md

@@ -3,11 +3,11 @@ title: "{{ frontmatter.title }}"
 date: "{{ frontmatter.date }}" #2021-06-10T10:46:33+02:00
 draft: false
 summary: "{{ frontmatter.summary }}"
-authors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
+contributors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
-categories: ["{{ frontmatter.card_type }}", "{{ frontmatter.feed_name}}"]
+card_type: "{{ frontmatter.card_type }}"
-contributors: ["{{ frontmatter.feed_name}}"]
+sources: ["{{ frontmatter.feed_name}}"]
 tags: {{ frontmatter.tags }}
 {% if frontmatter.featured_image %}featured_image: "{{frontmatter.featured_image}}"{% endif %}
 ---

konfluks/templates/hashtag.md

@@ -1,17 +1,27 @@
 ---
 date: {{ post_metadata.created_at }} #2021-06-10T10:46:33+02:00
 draft: false
-authors: ["{{ post_metadata.account.display_name }}"]
+contributors: ["{{ post_metadata.account.display_name }}"]
-contributors: ["{{ post_metadata.account.acct}}"]
 avatar: {{ post_metadata.account.avatar }}
-categories: ["shouts"]
-images: [{% for i in post_metadata.media_attachments %} {{ i.url }}, {% endfor %}]
 title: {{ post_metadata.account.display_name }}
 tags: [{% for i in post_metadata.tags %} "{{ i.name }}", {% endfor %}]
+images: [{% for i in post_metadata.media_attachments %}{% if i.type == "image" %}"{{ i.url | localize_media_url  }}", {%endif%}{% endfor %}]
+videos: [{% for i in post_metadata.media_attachments %}{% if i.type == "video" %}"{{ i.url | localize_media_url  }}", {%endif%}{% endfor %}]
 ---
 
 {% for item in post_metadata.media_attachments %}
+{% if item.type == "image" %}
 <img src="{{item.url | localize_media_url }}" alt="{{item.description}}">
+{% endif %}
+{% endfor %}
+
+{% for item in post_metadata.media_attachments %}
+{% if item.type == "video" %}
+<video controls width="540px" preload="none" poster="thumbnail.png">
+	<source src="{{item.url | localize_media_url }}" type="video/mp4">
+{% if item.description %}{{item.description}}{% endif %}
+</video>
+{% endif %}
 {% endfor %}
 
 {{ post_metadata.content | filter_mastodon_urls }}

konfluks/templates/timeline.md

@@ -3,10 +3,10 @@ title: "{{ frontmatter.title }}"
 date: "{{ frontmatter.date }}" #2021-06-10T10:46:33+02:00
 draft: false
 summary: "{{ frontmatter.summary }}"
-authors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
+contributors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
-categories: ["timeline", "{{ frontmatter.feed_name}}"]
+sources: ["timeline", "{{ frontmatter.feed_name}}"]
 timelines: {{ frontmatter.timelines }}
 hidden: true
 ---

konfluks/templates/video.md

@@ -9,7 +9,7 @@ channel_url: "{{ v.channel.url }}"
 contributors: ["{{ v.account.display_name }}"]
 preview_image: "{{ preview_image }}"
 images: ["./{{ preview_image }}"]
-categories: ["tv","{{ v.channel.display_name }}"]
+sources: ["{{ v.channel.display_name }}"]
 is_live: {{ v.is_live }}
 ---