ruangrupa/konfluks
ruangrupa/konfluks
2
0
Fork 2
Code Issues 9 Pull Requests Projects Releases Wiki Activity

Merge branch 'new-taxonomy'

Browse Source
This commit is contained in:
knoflook 2024-11-07 16:35:18 +01:00
commit 82a017f624
1 changed files with 55 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
konfluks/feed.py
View File

@ -156,7 +156,8 @@ def parse_enclosures(post_dir, entry):
print("found enclosed media", e.type) print("found enclosed media", e.type)
if "image/" in e.type: if "image/" in e.type:
featured_image = grab_media(post_dir, e.href) featured_image = grab_media(post_dir, e.href)
entry["featured_image"] = featured_image media_item = urlparse(e.href).path.split('/')[-1]
entry["featured_image"] = media_item
else: else:
print("FIXME:ignoring enclosed", e.type) print("FIXME:ignoring enclosed", e.type)
return entry return entry
@ -373,62 +374,66 @@ def main():
data = grab_feed(feed_url) data = grab_feed(feed_url)
if data: if data: #whenever we get a 200
if data.feed: #only if it is an actual feed
opds_feed = False
if 'links' in data.feed:
for i in data.feed['links']:
if i['rel'] == 'self':
if 'opds' in i['type']:
opds_feed = True
print("OPDS type feed!")
opds_feed = False for entry in data.entries:
for i in data.feed['links']: # if 'tags' in entry:
if i['rel'] == 'self': # for tag in entry.tags:
if 'opds' in i['type']: # for x in ['lumbung.space', 'D15', 'lumbung']:
opds_feed = True # if x in tag['term']:
print("OPDS type feed!") # print(entry.title)
entry["feed_name"] = feed_name
post_name = slugify(entry.title)
for entry in data.entries: # pixelfed returns the whole post text as the post name. max
# if 'tags' in entry: # filename length is 255 on many systems. here we're shortening
# for tag in entry.tags: # the name and adding a hash to it to avoid a conflict in a
# for x in ['lumbung.space', 'D15', 'lumbung']: # situation where 2 posts start with exactly the same text.
# if x in tag['term']: if len(post_name) > 150:
# print(entry.title) post_hash = md5(bytes(post_name, "utf-8"))
entry["feed_name"] = feed_name post_name = post_name[:150] + "-" + post_hash.hexdigest()
post_name = slugify(entry.title)
# pixelfed returns the whole post text as the post name. max
# filename length is 255 on many systems. here we're shortening
# the name and adding a hash to it to avoid a conflict in a
# situation where 2 posts start with exactly the same text.
if len(post_name) > 150:
post_hash = md5(bytes(post_name, "utf-8"))
post_name = post_name[:150] + "-" + post_hash.hexdigest()
if opds_feed:
entry['opds'] = True
#format: Beyond-Debiasing-Report_Online-75535a4886e3
post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1]
post_dir = os.path.join(output_dir, feed_name, post_name)
if post_name not in existing_posts:
# if there is a blog entry we dont already have, make it
if opds_feed: if opds_feed:
create_opds_post(post_dir, entry) entry['opds'] = True
else: #format: Beyond-Debiasing-Report_Online-75535a4886e3
create_post(post_dir, entry) post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1]
elif post_name in existing_posts: post_dir = os.path.join(output_dir, feed_name, post_name)
# if we already have it, update it
if opds_feed:
create_opds_post(post_dir, entry)
else:
create_post(post_dir, entry)
existing_posts.remove(
post_name
) # create list of posts which have not been returned by the feed
for post in existing_posts: if post_name not in existing_posts:
# remove blog posts no longer returned by the RSS feed # if there is a blog entry we dont already have, make it
print("deleted", post) if opds_feed:
shutil.rmtree(os.path.join(feed_dir, slugify(post))) create_opds_post(post_dir, entry)
else:
create_post(post_dir, entry)
elif post_name in existing_posts:
# if we already have it, update it
if opds_feed:
create_opds_post(post_dir, entry)
else:
create_post(post_dir, entry)
existing_posts.remove(
post_name
) # create list of posts which have not been returned by the feed
for post in existing_posts:
# remove blog posts no longer returned by the RSS feed
post_dir = os.path.join(output_dir, feed_name, post)
shutil.rmtree(post_dir)
print("deleted", post_dir)
else:
print(feed_url, "is not or no longer a feed!")
end = time.time() end = time.time()