feed: escape quotation marks

Reviewed-on: ruangrupa/lumbunglib#23

.gitignore

@@ -1,2 +1,7 @@
-test
+*.txt
+*.txt.*
 __pycache__
+etags
+test
+.venv
+content

README.md

@@ -1,3 +1,40 @@
 # lumbunglib
 
-> Python lib which powers `lumbung[dot]space` automation
+> Python lib which powers `lumbung.space` automation
+
+## hacking
+
+Install [poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions):
+
+```bash
+curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+```
+
+We use Poetry because it locks the dependencies all the way down and makes it
+easier to manage installation & maintenance in the long-term. Then install the
+dependencies & have them managed by Poetry:
+
+```
+poetry install
+```
+
+Each script requires some environment variables to run, you can see the latest
+deployment configuration over
+[here](https://git.autonomic.zone/ruangrupa/lumbung.space/src/branch/main/compose.yml),
+look for the values under the `environment: ...` stanza.
+
+All scripts have an entrypoint described in the
+[`pypoetry.toml`](https://git.autonomic.zone/ruangrupa/lumbunglib/src/commit/40bf9416b8792c08683ad8ac878093c7ef1b2f5d/pyproject.toml#L27-L31)
+which you can run via `poetry run ...`. For example, if you want to run the
+[`lumbunglib/video.py`](./lumbunglib/video.py) script, you'd do:
+
+```
+mkdir -p testdir
+export OUTPUT_DIR=/testdir
+poetry run lumbunglib-vid
+```
+
+Run `poetry run poetry2setup > setup.py` if updating the poetry dependencies.
+This allows us to run `pip install .` in the deployment and Pip will understand
+that it is just a regular Python package. If adding a new cli command, extend
+`pyproject.toml` with a new `[tool.poetry.scripts]` entry.

lumbunglib/cloudcal.py

@@ -3,6 +3,7 @@ import re
 import shutil
 from pathlib import Path
 from urllib.parse import urlparse
+from slugify import slugify
 
 import arrow
 import jinja2
@@ -75,6 +76,7 @@ def create_metadata(event):
         "duration": date.compress(event.duration),
         "location": event.location,
         "uid": event.uid,
+        "featured_image": "",
         "images": find_imageURLS(event.description),  # currently not used in template
     }
 
@@ -110,7 +112,6 @@ def localize_time(date):
         )
     return localized_begins
 
-
 def create_event_post(post_dir, event):
     """
     Create HUGO post based on calendar event metadata
@@ -146,12 +147,15 @@ def create_event_post(post_dir, event):
         if not os.path.exists(local_image):
             # download preview image
             response = requests.get(img, stream=True)
-            with open(local_image, "wb") as img_file:
-                shutil.copyfileobj(response.raw, img_file)
-                print('Downloaded image for event "{}"'.format(event.name))
-            event_metadata["description"] = event_metadata["description"].replace(
-                img, "![]({})".format(img_name)
-            )
+            if response.status_code == 200:
+                with open(local_image, "wb") as img_file:
+                    shutil.copyfileobj(response.raw, img_file)
+                    print('Downloaded image for event "{}"'.format(event.name))
+                event_metadata["description"] = event_metadata["description"].replace(
+                    img, "![]({})".format(img_name)
+                )
+                if event_metadata["featured_image"] == "":
+                    event_metadata["featured_image"] = img_name
         if img_name in existing_images:
             existing_images.remove(img_name)
 
@@ -184,18 +188,18 @@ def update_event_post(post_dir, event):
 
 def main():
     for event in list(cal.events):
+        post_name = slugify(event.name) + "-" + event.uid
+        post_dir = os.path.join(output_dir, post_name)
 
-        post_dir = os.path.join(output_dir, event.uid)
-
-        if event.uid not in existing_posts:
+        if post_name not in existing_posts:
             # if there is an event we dont already have, make it
             create_event_post(post_dir, event)
 
-        elif event.uid in existing_posts:
+        elif post_name in existing_posts:
             # if we already have it, update
             update_event_post(post_dir, event)
             existing_posts.remove(
-                event.uid
+                post_name
             )  # create list of posts which have not been returned by the calendar
 
     for post in existing_posts:

lumbunglib/feed.py

@@ -1,9 +1,11 @@
 import os
 import shutil
 import time
+from hashlib import md5
 from ast import literal_eval as make_tuple
 from pathlib import Path
 from urllib.parse import urlparse
+from re import sub
 
 import arrow
 import feedparser
@@ -56,10 +58,15 @@ def create_frontmatter(entry):
     published = arrow.get(published)
 
     if "author" in entry:
-        author = entry.author
+        author = sub('"', '\\"', entry.author)
     else:
         author = ""
 
+    if "title" in entry:
+        title = sub('"', '\\"', entry.title)
+    else:
+        title = ""
+
     tags = []
     if "tags" in entry:
         # TODO finish categories
@@ -67,7 +74,7 @@ def create_frontmatter(entry):
             tags.append(t["term"])
 
     frontmatter = {
-        "title": entry.title,
+        "title": title,
         "date": published.format(),
         "summary": "",
         "author": author,
@@ -194,13 +201,25 @@ def main():
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
+    feed_dict = dict()
+    for url in feed_urls:
+        feed_name = urlparse(url).netloc
+        feed_dict[url] = feed_name
+
+    feed_names = feed_dict.values()
+    content_dirs = os.listdir(output_dir)
+    for i in content_dirs:
+        if i not in feed_names:
+            shutil.rmtree(os.path.join(output_dir, i))
+            print("%s not in feeds_list.txt, removing local data" %(i))
+
     # add iframe to the allowlist of feedparser's sanitizer,
     # this is now handled in parse_post()
     feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {"iframe"}
 
     for feed_url in feed_urls:
 
-        feed_name = urlparse(feed_url).netloc
+        feed_name = feed_dict[feed_url]
 
         feed_dir = os.path.join(output_dir, feed_name)
 
@@ -221,6 +240,13 @@ def main():
                 entry["feed_name"] = feed_name
 
                 post_name = slugify(entry.title)
+                # pixelfed returns the whole post text as the post name. max
+                # filename length is 255 on many systems. here we're shortening
+                # the name and adding a hash to it to avoid a conflict in a
+                # situation where 2 posts start with exactly the same text.
+                if len(post_name) > 150:
+                    post_hash = md5(bytes(post_name, "utf-8"))
+                    post_name = post_name[:150] + "-" + post_hash.hexdigest()
                 post_dir = os.path.join(output_dir, feed_name, post_name)
 
                 if post_name not in existing_posts:

lumbunglib/hashtag.py

@@ -1,6 +1,7 @@
 import os
 import shutil
 from pathlib import Path
+from re import sub
 
 import jinja2
 import requests
@@ -9,20 +10,22 @@ from mastodon import Mastodon
 instance = "https://social.lumbung.space"
 email = ""
 password = ""
-hashtags = ["jalansesama"]
+hashtags = [
+    "documentafifteen",
+    "harvestedbyputra",
+    "jalansesama",
+    "lumbungdotspace",
+    "majelisakakbar",
+    "majelisakbar",
+    "warungkopi",
+    "lumbungkios",
+]
 
 
 def login_mastodon_bot():
     mastodon = Mastodon(
-        client_id="publishbot_clientcred.secret",
-        api_base_url=instance,
-    )
-
-    mastodon.log_in(
-        email,
-        password,
-        to_file="publishbot_usercred.secret",
-        scopes=["read"],
+        access_token=os.environ.get("MASTODON_AUTH_TOKEN"),
+        api_base_url = instance
     )
 
     return mastodon
@@ -66,7 +69,9 @@ def create_post(post_directory, post_metadata):
 
     template_dir = os.path.join(Path(__file__).parent.resolve(), "templates")
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir))
-
+    name = post_metadata['account']['display_name']
+    name = sub('"', '\\"', name)
+    post_metadata['account']['display_name'] = name
     env.filters["localize_media_url"] = localize_media_url
     env.filters["filter_mastodon_urls"] = filter_mastodon_urls
 
@@ -98,11 +103,14 @@ def filter_mastodon_urls(content):
 
 def main():
     mastodon = login_mastodon_bot()
-
     output_dir = os.environ.get("OUTPUT_DIR")
     if not os.path.exists(output_dir):
         os.mkdir(output_dir)
 
+    all_existing_posts = []
+    for i in os.listdir(output_dir):
+        all_existing_posts += os.listdir(os.path.join(output_dir, i))
+
     for hashtag in hashtags:
 
         hashtag_dir = os.path.join(output_dir, hashtag)
@@ -120,14 +128,15 @@ def main():
 
         for post_metadata in timeline:
             post_dir = os.path.join(hashtag_dir, str(post_metadata["id"]))
-
             # if there is a post in the feed we dont already have locally, make it
-            if str(post_metadata["id"]) not in existing_posts:
-
+            if str(post_metadata["id"]) not in all_existing_posts:
                 if not post_metadata[
                     "local_only"
                 ]:  # if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature
                     create_post(post_dir, post_metadata)
+                    all_existing_posts.append(str(post_metadata["id"]))
+                else:
+                    print("not pulling post %s (post is local only)" % (post_metadata["id"]))
 
             # if we already have the post do nothing, possibly update
             elif str(post_metadata["id"]) in existing_posts:
@@ -135,6 +144,8 @@ def main():
                 existing_posts.remove(
                     str(post_metadata["id"])
                 )  # create list of posts which have not been returned in the feed
+            elif str(post_metadata["id"]) in all_existing_posts:
+                print("skipping post %s as it was already pulled with a different hashtag." % (str(post_metadata["id"])))
 
         for post in existing_posts:
             print(

lumbunglib/templates/calendar.md

@@ -8,6 +8,9 @@ event_end: "{{ event.end }}"
 duration: "{{ event.duration }}"
 localized_begin: "{{ event.localized_begin }}"
 uid: "{{ event.uid }}"
+{% if event.featured_image %}
+featured_image: "{{ event.featured_image }}"
+{% endif %}
 {% if event.location %}
 location: "{{ event.location }}"
 {% endif %}

lumbunglib/templates/feed.md

@@ -7,7 +7,7 @@ author: "{{ frontmatter.author }}"
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
 categories: ["network", "{{ frontmatter.feed_name}}"]
-tags: { { frontmatter.tags } }
+tags: {{ frontmatter.tags }}
 ---
 
 {{ content }}

lumbunglib/templates/hashtag.md

@@ -4,6 +4,8 @@ draft: false
 author: "{{ post_metadata.account.display_name }}"
 avatar: "{{ post_metadata.account.avatar }}"
 categories: ["shouts"]
+images: [{% for i in post_metadata.media_attachments %} "{{ i.url }}", {% endfor %}]
+title: "{{ post_metadata.account.display_name }}"
 tags: [{% for i in post_metadata.tags %} "{{ i.name }}", {% endfor %}]
 ---
 

lumbunglib/templates/video.md

@@ -7,6 +7,7 @@ video_duration: "{{ v.duration | duration }} "
 video_channel: "{{ v.channel.display_name }}"
 channel_url: "{{ v.channel.url }}"
 preview_image: "{{ preview_image }}"
+images: ["./{{ preview_image }}"]
 categories: ["tv","{{ v.channel.display_name }}"]
 is_live: {{ v.is_live }}
 ---

lumbunglib/video.py

@@ -4,6 +4,7 @@ import json
 import os
 import shutil
 from pathlib import Path
+from slugify import slugify
 
 import arrow
 import jinja2
@@ -97,14 +98,28 @@ def update_post(post_directory, video_metadata, host):
             # compat for when there is no timestamp yet..
             create_post(post_directory, video_metadata, host)
 
-
 def main():
     v = peertube.VideoApi(client)
-
-    response = v.videos_get(count=100, filter="local", tags_one_of="publish")
+    count = 100
+    page = 0
+    response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
 
     videos = response.to_dict()
-    videos = videos["data"]
+    total = videos['total']
+    videos = videos['data']
+    total -= count
+    if total > 0:
+        to_download = total // count
+        last_page = total % count
+        for i in range(to_download):
+            page += 1
+            response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
+            videos += response.to_dict()['data']
+        if last_page > 0:
+            page += 1
+            response = v.videos_get(count=count, filter="local", tags_one_of="publish", start=page)
+            videos += response.to_dict()['data'][-1*last_page:]
+
 
     output_dir = os.environ.get("OUTPUT_DIR")
 
@@ -114,10 +129,11 @@ def main():
     existing_posts = os.listdir(output_dir)
 
     for video_metadata in videos:
-        post_dir = os.path.join(output_dir, video_metadata["uuid"])
+        post_name = slugify(video_metadata["name"]) + "-" + video_metadata["uuid"]
+        post_dir = os.path.join(output_dir, post_name)
 
         if (
-            video_metadata["uuid"] not in existing_posts
+            post_name not in existing_posts
         ):  # if there is a video we dont already have, make it
             print(
                 "New: ", video_metadata["name"], "({})".format(video_metadata["uuid"])
@@ -125,11 +141,11 @@ def main():
             create_post(post_dir, video_metadata, host)
 
         elif (
-            video_metadata["uuid"] in existing_posts
+            post_name in existing_posts
         ):  # if we already have the video do nothing, possibly update
             update_post(post_dir, video_metadata, host)
             existing_posts.remove(
-                video_metadata["uuid"]
+                post_name
             )  # create list of posts which have not been returned by peertube
 
     for post in existing_posts:

poetry.lock

@@ -9,6 +9,43 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 [package.dependencies]
 python-dateutil = "*"
 
+[[package]]
+name = "beautifulsoup4"
+version = "4.10.0"
+description = "Screen-scraping library"
+category = "main"
+optional = false
+python-versions = ">3.0.0"
+
+[package.dependencies]
+soupsieve = ">1.2"
+
+[package.extras]
+html5lib = ["html5lib"]
+lxml = ["lxml"]
+
+[[package]]
+name = "blurhash"
+version = "1.1.4"
+description = "Pure-Python implementation of the blurhash algorithm."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.extras]
+test = ["pillow", "numpy", "pytest"]
+
+[[package]]
+name = "bs4"
+version = "0.0.1"
+description = "Dummy package for Beautiful Soup"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+beautifulsoup4 = "*"
+
 [[package]]
 name = "certifi"
 version = "2021.10.8"
@@ -19,7 +56,7 @@ python-versions = "*"
 
 [[package]]
 name = "charset-normalizer"
-version = "2.0.9"
+version = "2.0.10"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
 category = "main"
 optional = false
@@ -28,6 +65,25 @@ python-versions = ">=3.5.0"
 [package.extras]
 unicode_backport = ["unicodedata2"]
 
+[[package]]
+name = "decorator"
+version = "5.1.1"
+description = "Decorators for Humans"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "feedparser"
+version = "6.0.8"
+description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+sgmllib3k = "*"
+
 [[package]]
 name = "ics"
 version = "0.7"
@@ -72,6 +128,28 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "mastodon.py"
+version = "1.5.1"
+description = "Python wrapper for the Mastodon API"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+blurhash = ">=1.1.4"
+decorator = ">=4.0.0"
+python-dateutil = "*"
+python-magic = "*"
+pytz = "*"
+requests = ">=2.4.2"
+six = "*"
+
+[package.extras]
+blurhash = ["blurhash (>=1.1.4)"]
+test = ["blurhash (>=1.1.4)", "cryptography (>=1.6.0)", "http-ece (>=1.0.5)", "pytest", "pytest-cov", "pytest-mock", "pytest-runner", "pytest-vcr", "requests-mock", "vcrpy"]
+webpush = ["cryptography (>=1.6.0)", "http-ece (>=1.0.5)"]
+
 [[package]]
 name = "natural"
 version = "0.2.0"
@@ -134,6 +212,14 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "python-magic"
+version = "0.4.24"
+description = "File type identification using libmagic"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
 [[package]]
 name = "python-slugify"
 version = "5.0.2"
@@ -148,9 +234,17 @@ text-unidecode = ">=1.3"
 [package.extras]
 unidecode = ["Unidecode (>=1.1.1)"]
 
+[[package]]
+name = "pytz"
+version = "2021.3"
+description = "World timezone definitions, modern and historical"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "requests"
-version = "2.26.0"
+version = "2.27.1"
 description = "Python HTTP for Humans."
 category = "main"
 optional = false
@@ -166,6 +260,14 @@ urllib3 = ">=1.21.1,<1.27"
 socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
 use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
 
+[[package]]
+name = "sgmllib3k"
+version = "1.0.0"
+description = "Py3k port of sgmllib."
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -174,6 +276,14 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 
+[[package]]
+name = "soupsieve"
+version = "2.3.1"
+description = "A modern CSS selector implementation for Beautiful Soup."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "tatsu"
 version = "5.7.0"
@@ -195,7 +305,7 @@ python-versions = "*"
 
 [[package]]
 name = "urllib3"
-version = "1.26.7"
+version = "1.26.8"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 category = "main"
 optional = false
@@ -209,20 +319,39 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "e355abc98d6deaf8a6d6a3567394dec76c33e05212e4913b0d03173f137ca43b"
+content-hash = "c5c987253f949737210f4a3d3c3c24b0affd4a9c7d06de386c9bd514c592db8b"
 
 [metadata.files]
 arrow = [
     {file = "arrow-0.14.7-py2.py3-none-any.whl", hash = "sha256:4bfacea734ead51495dc47df00421ecfd4ca1f2c0fbe58b9a26eaeddedc31caf"},
     {file = "arrow-0.14.7.tar.gz", hash = "sha256:67f8be7c0cf420424bc62d8d7dc40b44e4bb2f7b515f9cc2954fb36e35797656"},
 ]
+beautifulsoup4 = [
+    {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"},
+    {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"},
+]
+blurhash = [
+    {file = "blurhash-1.1.4-py2.py3-none-any.whl", hash = "sha256:7611c1bc41383d2349b6129208587b5d61e8792ce953893cb49c38beeb400d1d"},
+    {file = "blurhash-1.1.4.tar.gz", hash = "sha256:da56b163e5a816e4ad07172f5639287698e09d7f3dc38d18d9726d9c1dbc4cee"},
+]
+bs4 = [
+    {file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"},
+]
 certifi = [
     {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
     {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
 ]
 charset-normalizer = [
-    {file = "charset-normalizer-2.0.9.tar.gz", hash = "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"},
-    {file = "charset_normalizer-2.0.9-py3-none-any.whl", hash = "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721"},
+    {file = "charset-normalizer-2.0.10.tar.gz", hash = "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd"},
+    {file = "charset_normalizer-2.0.10-py3-none-any.whl", hash = "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"},
+]
+decorator = [
+    {file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
+    {file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
+]
+feedparser = [
+    {file = "feedparser-6.0.8-py3-none-any.whl", hash = "sha256:1b7f57841d9cf85074deb316ed2c795091a238adb79846bc46dccdaf80f9c59a"},
+    {file = "feedparser-6.0.8.tar.gz", hash = "sha256:5ce0410a05ab248c8c7cfca3a0ea2203968ee9ff4486067379af4827a59f9661"},
 ]
 ics = [
     {file = "ics-0.7-py2.py3-none-any.whl", hash = "sha256:bf5fbdef6e1e073afdadf1b996f0271186dd114a148e38e795919a1ae644d6ac"},
@@ -238,12 +367,28 @@ jinja2 = [
     {file = "Jinja2-3.0.3.tar.gz", hash = "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"},
 ]
 markupsafe = [
+    {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"},
+    {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f"},
+    {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"},
     {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"},
     {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"},
@@ -252,14 +397,27 @@ markupsafe = [
     {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f"},
     {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2"},
     {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9"},
+    {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86"},
     {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"},
     {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194"},
+    {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"},
     {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"},
     {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"},
@@ -269,10 +427,20 @@ markupsafe = [
     {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135"},
     {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902"},
     {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e"},
+    {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1"},
     {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"},
     {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"},
     {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"},
 ]
+"mastodon.py" = [
+    {file = "Mastodon.py-1.5.1-py2.py3-none-any.whl", hash = "sha256:cc454cac0ed1ae4f105f7399ea53f5b31a1be5075d1882f47162d2e78a9e4064"},
+    {file = "Mastodon.py-1.5.1.tar.gz", hash = "sha256:2afddbad8b5d7326fcc8a8f8c62bfe956e34627f516b06c6694fc8c8fedc33ee"},
+]
 natural = [
     {file = "natural-0.2.0.tar.gz", hash = "sha256:18c83662d2d33fd7e6eee4e3b0d7366e1ce86225664e3127a2aaf0a3233f7df2"},
 ]
@@ -289,18 +457,33 @@ python-dateutil = [
     {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
     {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
 ]
+python-magic = [
+    {file = "python-magic-0.4.24.tar.gz", hash = "sha256:de800df9fb50f8ec5974761054a708af6e4246b03b4bdaee993f948947b0ebcf"},
+    {file = "python_magic-0.4.24-py2.py3-none-any.whl", hash = "sha256:4fec8ee805fea30c07afccd1592c0f17977089895bdfaae5fec870a84e997626"},
+]
 python-slugify = [
     {file = "python-slugify-5.0.2.tar.gz", hash = "sha256:f13383a0b9fcbe649a1892b9c8eb4f8eab1d6d84b84bb7a624317afa98159cab"},
     {file = "python_slugify-5.0.2-py2.py3-none-any.whl", hash = "sha256:6d8c5df75cd4a7c3a2d21e257633de53f52ab0265cd2d1dc62a730e8194a7380"},
 ]
+pytz = [
+    {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"},
+    {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"},
+]
 requests = [
-    {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"},
-    {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"},
+    {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
+    {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
+]
+sgmllib3k = [
+    {file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},
 ]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
+soupsieve = [
+    {file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"},
+    {file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"},
+]
 tatsu = [
     {file = "TatSu-5.7.0-py2.py3-none-any.whl", hash = "sha256:9eebadfc2889d8e82e197df22913df56ff204bf4cfc62db49a5c7edd084e10b4"},
     {file = "TatSu-5.7.0.zip", hash = "sha256:428136cd4aa9600fcd01428bd5667fc752062f54bd0148dc1e64fee7b8d05fa4"},
@@ -310,6 +493,6 @@ text-unidecode = [
     {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"},
 ]
 urllib3 = [
-    {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"},
-    {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"},
+    {file = "urllib3-1.26.8-py2.py3-none-any.whl", hash = "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed"},
+    {file = "urllib3-1.26.8.tar.gz", hash = "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"},
 ]

pyproject.toml

@@ -13,6 +13,9 @@ natural = "^0.2.0"
 python-slugify = "^5.0.2"
 requests = "^2.26.0"
 peertube = {git = "https://framagit.org/framasoft/peertube/clients/python.git"}
+feedparser = "^6.0.8"
+bs4 = "^0.0.1"
+"Mastodon.py" = "^1.5.1"
 
 [tool.poetry.dev-dependencies]
 poetry2setup = "^1.0.0"
@@ -24,3 +27,5 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry.scripts]
 lumbunglib-cal = "lumbunglib.cloudcal:main"
 lumbunglib-vid = "lumbunglib.video:main"
+lumbunglib-feed = "lumbunglib.feed:main"
+lumbunglib-hash = "lumbunglib.hashtag:main"

setup.py

@@ -1,40 +1,47 @@
 # -*- coding: utf-8 -*-
 from setuptools import setup
 
-packages = ["lumbunglib"]
+packages = \
+['lumbunglib']
 
-package_data = {"": ["*"], "lumbunglib": ["templates/*"]}
+package_data = \
+{'': ['*'], 'lumbunglib': ['templates/*']}
 
-install_requires = [
-    "Jinja2>=3.0.3,<4.0.0",
-    "ics>=0.7,<0.8",
-    "natural>=0.2.0,<0.3.0",
-    "peertube @ "
-    "git+https://framagit.org/framasoft/peertube/clients/python.git@master",
-    "python-slugify>=5.0.2,<6.0.0",
-    "requests>=2.26.0,<3.0.0",
-]
+install_requires = \
+['Jinja2>=3.0.3,<4.0.0',
+ 'Mastodon.py>=1.5.1,<2.0.0',
+ 'bs4>=0.0.1,<0.0.2',
+ 'feedparser>=6.0.8,<7.0.0',
+ 'ics>=0.7,<0.8',
+ 'natural>=0.2.0,<0.3.0',
+ 'peertube @ '
+ 'git+https://framagit.org/framasoft/peertube/clients/python.git@master',
+ 'python-slugify>=5.0.2,<6.0.0',
+ 'requests>=2.26.0,<3.0.0']
 
-entry_points = {
-    "console_scripts": ["cal = lumbunglib.cloudcal:main", "vid = lumbunglib.video:main"]
-}
+entry_points = \
+{'console_scripts': ['lumbunglib-cal = lumbunglib.cloudcal:main',
+                     'lumbunglib-feed = lumbunglib.feed:main',
+                     'lumbunglib-hash = lumbunglib.hashtag:main',
+                     'lumbunglib-vid = lumbunglib.video:main']}
 
 setup_kwargs = {
-    "name": "lumbunglib",
-    "version": "0.1.0",
-    "description": "Python lib which powers lumbung[dot]space automation",
-    "long_description": None,
-    "author": "rra",
-    "author_email": None,
-    "maintainer": None,
-    "maintainer_email": None,
-    "url": None,
-    "packages": packages,
-    "package_data": package_data,
-    "install_requires": install_requires,
-    "entry_points": entry_points,
-    "python_requires": ">=3.9,<4.0",
+    'name': 'lumbunglib',
+    'version': '0.1.0',
+    'description': 'Python lib which powers lumbung[dot]space automation',
+    'long_description': None,
+    'author': 'rra',
+    'author_email': None,
+    'maintainer': None,
+    'maintainer_email': None,
+    'url': None,
+    'packages': packages,
+    'package_data': package_data,
+    'install_requires': install_requires,
+    'entry_points': entry_points,
+    'python_requires': '>=3.9,<4.0',
 }
 
 
 setup(**setup_kwargs)
+