ruangrupa/konfluks
ruangrupa/konfluks
2
0
Fork 2
Code Issues 9 Pull Requests Projects Releases Wiki Activity

Compare commits

base: ruangrupa:82a017f62425534f1cdc2442ac5b6a154b66f0b3
Branches Tags
ruangrupa:main
ruangrupa:opds-fetchall
ruangrupa:new-taxonomy
..
compare: ruangrupa:3055ee37df66003251746f2b847f34a272fa3d9a
Branches Tags
ruangrupa:opds-fetchall
ruangrupa:main
ruangrupa:new-taxonomy

No commits in common. "82a017f62425534f1cdc2442ac5b6a154b66f0b3" and "3055ee37df66003251746f2b847f34a272fa3d9a" have entirely different histories.
82a017f624 ... 3055ee37df

1 changed files with 50 additions and 55 deletions
Show Stats Expand all files Collapse all files

105
konfluks/feed.py
View File

@ -156,8 +156,7 @@ def parse_enclosures(post_dir, entry):
print("found enclosed media", e.type) print("found enclosed media", e.type)
if "image/" in e.type: if "image/" in e.type:
featured_image = grab_media(post_dir, e.href) featured_image = grab_media(post_dir, e.href)
media_item = urlparse(e.href).path.split('/')[-1] entry["featured_image"] = featured_image
entry["featured_image"] = media_item
else: else:
print("FIXME:ignoring enclosed", e.type) print("FIXME:ignoring enclosed", e.type)
return entry return entry
@ -374,66 +373,62 @@ def main():
data = grab_feed(feed_url) data = grab_feed(feed_url)
if data: #whenever we get a 200 if data:
if data.feed: #only if it is an actual feed
opds_feed = False
if 'links' in data.feed:
for i in data.feed['links']:
if i['rel'] == 'self':
if 'opds' in i['type']:
opds_feed = True
print("OPDS type feed!")
for entry in data.entries: opds_feed = False
# if 'tags' in entry: for i in data.feed['links']:
# for tag in entry.tags: if i['rel'] == 'self':
# for x in ['lumbung.space', 'D15', 'lumbung']: if 'opds' in i['type']:
# if x in tag['term']: opds_feed = True
# print(entry.title) print("OPDS type feed!")
entry["feed_name"] = feed_name
post_name = slugify(entry.title)
# pixelfed returns the whole post text as the post name. max for entry in data.entries:
# filename length is 255 on many systems. here we're shortening # if 'tags' in entry:
# the name and adding a hash to it to avoid a conflict in a # for tag in entry.tags:
# situation where 2 posts start with exactly the same text. # for x in ['lumbung.space', 'D15', 'lumbung']:
if len(post_name) > 150: # if x in tag['term']:
post_hash = md5(bytes(post_name, "utf-8")) # print(entry.title)
post_name = post_name[:150] + "-" + post_hash.hexdigest() entry["feed_name"] = feed_name
post_name = slugify(entry.title)
# pixelfed returns the whole post text as the post name. max
# filename length is 255 on many systems. here we're shortening
# the name and adding a hash to it to avoid a conflict in a
# situation where 2 posts start with exactly the same text.
if len(post_name) > 150:
post_hash = md5(bytes(post_name, "utf-8"))
post_name = post_name[:150] + "-" + post_hash.hexdigest()
if opds_feed:
entry['opds'] = True
#format: Beyond-Debiasing-Report_Online-75535a4886e3
post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1]
post_dir = os.path.join(output_dir, feed_name, post_name)
if post_name not in existing_posts:
# if there is a blog entry we dont already have, make it
if opds_feed: if opds_feed:
entry['opds'] = True create_opds_post(post_dir, entry)
#format: Beyond-Debiasing-Report_Online-75535a4886e3 else:
post_name = slugify(entry['title'])+'-'+entry['id'].split('-')[-1] create_post(post_dir, entry)
post_dir = os.path.join(output_dir, feed_name, post_name) elif post_name in existing_posts:
# if we already have it, update it
if opds_feed:
create_opds_post(post_dir, entry)
else:
create_post(post_dir, entry)
existing_posts.remove(
post_name
) # create list of posts which have not been returned by the feed
if post_name not in existing_posts: for post in existing_posts:
# if there is a blog entry we dont already have, make it # remove blog posts no longer returned by the RSS feed
if opds_feed: print("deleted", post)
create_opds_post(post_dir, entry) shutil.rmtree(os.path.join(feed_dir, slugify(post)))
else:
create_post(post_dir, entry)
elif post_name in existing_posts:
# if we already have it, update it
if opds_feed:
create_opds_post(post_dir, entry)
else:
create_post(post_dir, entry)
existing_posts.remove(
post_name
) # create list of posts which have not been returned by the feed
for post in existing_posts:
# remove blog posts no longer returned by the RSS feed
post_dir = os.path.join(output_dir, feed_name, post)
shutil.rmtree(post_dir)
print("deleted", post_dir)
else:
print(feed_url, "is not or no longer a feed!")
end = time.time() end = time.time()