follow navigation links in opds to retrieve entire catalog

Reviewed-on: #43

konfluks/feed.py

@@ -155,8 +155,11 @@ def parse_enclosures(post_dir, entry):
         if "type" in e:
             print("found enclosed media", e.type)
             if "image/" in e.type:
+                if not os.path.exists(post_dir): #this might be redundant with create_post
+                    os.makedirs(post_dir)
                 featured_image = grab_media(post_dir, e.href)
-                entry["featured_image"] = featured_image
+                media_item = urlparse(e.href).path.split('/')[-1]
+                entry["featured_image"] = media_item
             else:
                 print("FIXME:ignoring enclosed", e.type)
     return entry
@@ -309,12 +312,15 @@ def create_opds_post(post_dir, entry):
         ft = item['type'].split('/')[-1]
         fn = item['rel'].split('/')[-1]
 
-        if fn == "acquisition":
-            fn = "publication" #calling the publications acquisition is weird
+        # entry.links has image, thumbnail and publication/acquisition.
+        # Only downloading image for now
+        #if fn == "acquisition": 
+        #fn = "publication" #calling the publications acquisition is weird 
 
-        prefered_name = "{}-{}.{}".format(fn, slugify(entry['title']), ft)
-
-        grab_media(post_dir, item['href'], prefered_name)
+        if 'image' in fn:
+            prefered_name = "{}-{}.{}".format(fn, slugify(entry['title']), ft)
+            grab_media(post_dir, item['href'], prefered_name)
+            frontmatter['featured_image'] = prefered_name
 
         if "summary" in entry:
             summary = entry.summary
@@ -330,6 +336,18 @@ def create_opds_post(post_dir, entry):
         timestamp = arrow.get(entry['updated_parsed'])
         f.write(timestamp.format('X'))
 
+def opds_fetch_more(data):
+    """
+    Look for mode OPDS feeds to pull, untill we no longer hit the "next" navigation property. 
+    """
+    for link in data.feed.links:
+        for i in link:
+            if link[i] == 'next':
+                print(link['href'])
+                data = grab_feed(link['href'])
+                return data
+    return None
+
 
 def main():
     feed_urls = open("feeds_list.txt", "r").read().splitlines()
@@ -373,62 +391,77 @@ def main():
 
         data = grab_feed(feed_url)
 
-        if data:
-
-            opds_feed = False
-            for i in data.feed['links']:
-                if i['rel'] == 'self':
-                    if 'opds' in i['type']:
-                        opds_feed = True
-                        print("OPDS type feed!")
+        if data: #whenever we get a 200
+            if data.feed: #only if it is an actual feed
+                opds_feed = False
+                opds_entries = []
+                if 'links' in data.feed:
+                    for i in data.feed['links']:
+                            if i['rel'] == 'self':
+                                if 'opds' in i['type']:
+                                    opds_feed = True
+                                    print("OPDS type feed!")
+                                    feed_data = data
+                                    while feed_data:
+                                        feed_data = opds_fetch_more(feed_data)
+                                        if feed_data:
+                                            for i in feed_data.entries:
+                                                opds_entries.append(i)
+                                    for i in opds_entries:
+                                        data['entries'].append(i)
+                                    
 
 
-            for entry in data.entries:
-                # if 'tags' in entry:
-                #     for tag in entry.tags:
-                #        for x in ['lumbung.space', 'D15', 'lumbung']:
-                #            if x in tag['term']:
-                #                print(entry.title)
-                entry["feed_name"] = feed_name
+                for entry in data.entries:
+                    # if 'tags' in entry:
+                    #     for tag in entry.tags:
+                    #        for x in ['lumbung.space', 'D15', 'lumbung']:
+                    #            if x in tag['term']:
+                    #                print(entry.title)
+                    entry["feed_name"] = feed_name
 
-                post_name = slugify(entry.title)
+                    post_name = slugify(entry.title)
 
-                # pixelfed returns the whole post text as the post name. max
-                # filename length is 255 on many systems. here we're shortening
-                # the name and adding a hash to it to avoid a conflict in a
-                # situation where 2 posts start with exactly the same text.
-                if len(post_name) > 150:
-                    post_hash = md5(bytes(post_name, "utf-8"))
-                    post_name = post_name[:150] + "-" + post_hash.hexdigest()
+                    # pixelfed returns the whole post text as the post name. max
+                    # filename length is 255 on many systems. here we're shortening
+                    # the name and adding a hash to it to avoid a conflict in a
+                    # situation where 2 posts start with exactly the same text.
+                    if len(post_name) > 150:
+                        post_hash = md5(bytes(post_name, "utf-8"))
+                        post_name = post_name[:150] + "-" + post_hash.hexdigest()
 
-                if opds_feed:
-                    entry['opds'] = True
-                    #format: Beyond-Debiasing-Report_Online-75535a4886e3
-                    post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1]
-
-                post_dir = os.path.join(output_dir, feed_name, post_name)
-
-                if post_name not in existing_posts:
-                    # if there is a blog entry we dont already have, make it
                     if opds_feed:
-                        create_opds_post(post_dir, entry)
-                    else:
-                        create_post(post_dir, entry)
+                        entry['opds'] = True
+                        #format: Beyond-Debiasing-Report_Online-75535a4886e3
+                        post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1]
 
-                elif post_name in existing_posts:
-                    # if we already have it, update it
-                    if opds_feed:
-                        create_opds_post(post_dir, entry)
-                    else:
-                        create_post(post_dir, entry)
-                    existing_posts.remove(
-                        post_name
-                    )  # create list of posts which have not been returned by the feed
+                    post_dir = os.path.join(output_dir, feed_name, post_name)
 
-            for post in existing_posts:
-                # remove blog posts no longer returned by the RSS feed
-                print("deleted", post)
-                shutil.rmtree(os.path.join(feed_dir, slugify(post)))
+                    if post_name not in existing_posts:
+                        # if there is a blog entry we dont already have, make it
+                        if opds_feed:
+                            create_opds_post(post_dir, entry)
+                        else:
+                            create_post(post_dir, entry)
+
+                    elif post_name in existing_posts:
+                        # if we already have it, update it
+                        if opds_feed:
+                            create_opds_post(post_dir, entry)
+                        else:
+                            create_post(post_dir, entry)
+                        existing_posts.remove(
+                            post_name
+                        )  # create list of posts which have not been returned by the feed
+
+
+                for post in existing_posts:
+                    # remove blog posts no longer returned by the RSS feed
+                    post_dir = os.path.join(output_dir, feed_name, post)
+                    shutil.rmtree(post_dir)
+                    print("deleted", post_dir)
+            else:
+                print(feed_url, "is not or no longer a feed!")
 
     end = time.time()
 

konfluks/templates/calendar.md

@@ -2,7 +2,7 @@
 title: "{{ event.name }}"
 date: "{{ event.begin }}" #2021-06-10T10:46:33+02:00
 draft: false
-categories: "calendar"
+source: "lumbung calendar"
 event_begin: "{{ event.begin }}"
 event_end: "{{ event.end }}"
 duration: "{{ event.duration }}"

konfluks/templates/feed.md

@@ -3,11 +3,11 @@ title: "{{ frontmatter.title }}"
 date: "{{ frontmatter.date }}" #2021-06-10T10:46:33+02:00
 draft: false
 summary: "{{ frontmatter.summary }}"
-authors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
+contributors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
-categories: ["{{ frontmatter.card_type }}", "{{ frontmatter.feed_name}}"]
-contributors: ["{{ frontmatter.feed_name}}"]
+card_type: "{{ frontmatter.card_type }}"
+sources: ["{{ frontmatter.feed_name}}"]
 tags: {{ frontmatter.tags }}
 {% if frontmatter.featured_image %}featured_image: "{{frontmatter.featured_image}}"{% endif %}
 ---

konfluks/templates/hashtag.md

@@ -1,10 +1,8 @@
 ---
 date: {{ post_metadata.created_at }} #2021-06-10T10:46:33+02:00
 draft: false
-authors: ["{{ post_metadata.account.display_name }}"]
-contributors: ["{{ post_metadata.account.acct}}"]
+contributors: ["{{ post_metadata.account.display_name }}"]
 avatar: {{ post_metadata.account.avatar }}
-categories: ["shouts"]
 title: {{ post_metadata.account.display_name }}
 tags: [{% for i in post_metadata.tags %} "{{ i.name }}", {% endfor %}]
 images: [{% for i in post_metadata.media_attachments %}{% if i.type == "image" %}"{{ i.url | localize_media_url  }}", {%endif%}{% endfor %}]

konfluks/templates/timeline.md

@@ -3,10 +3,10 @@ title: "{{ frontmatter.title }}"
 date: "{{ frontmatter.date }}" #2021-06-10T10:46:33+02:00
 draft: false
 summary: "{{ frontmatter.summary }}"
-authors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
+contributors: {% if frontmatter.author %} ["{{ frontmatter.author }}"] {% endif %}
 original_link: "{{ frontmatter.original_link }}"
 feed_name: "{{ frontmatter.feed_name}}"
-categories: ["timeline", "{{ frontmatter.feed_name}}"]
+sources: ["timeline", "{{ frontmatter.feed_name}}"]
 timelines: {{ frontmatter.timelines }}
 hidden: true
 ---

konfluks/templates/video.md

@@ -9,7 +9,7 @@ channel_url: "{{ v.channel.url }}"
 contributors: ["{{ v.account.display_name }}"]
 preview_image: "{{ preview_image }}"
 images: ["./{{ preview_image }}"]
-categories: ["tv","{{ v.channel.display_name }}"]
+sources: ["{{ v.channel.display_name }}"]
 is_live: {{ v.is_live }}
 ---