ruangrupa/konfluks
ruangrupa/konfluks
2
0
Fork 2
Code Issues 9 Pull Requests Projects Releases Wiki Activity

Compare commits

base: ruangrupa:opds-fetchall
Branches Tags
ruangrupa:main
ruangrupa:opds-fetchall
ruangrupa:new-taxonomy
..
compare: ruangrupa:main
Branches Tags
ruangrupa:opds-fetchall
ruangrupa:main
ruangrupa:new-taxonomy

No commits in common. "opds-fetchall" and "main" have entirely different histories.
opds-fetch ... main

1 changed files with 5 additions and 31 deletions
Show Stats Expand all files Collapse all files

36
konfluks/feed.py
View File

@ -312,15 +312,12 @@ def create_opds_post(post_dir, entry):
ft = item['type'].split('/')[-1] ft = item['type'].split('/')[-1]
fn = item['rel'].split('/')[-1] fn = item['rel'].split('/')[-1]
# entry.links has image, thumbnail and publication/acquisition. if fn == "acquisition":
# Only downloading image for now fn = "publication" #calling the publications acquisition is weird
#if fn == "acquisition":
#fn = "publication" #calling the publications acquisition is weird
if 'image' in fn: prefered_name = "{}-{}.{}".format(fn, slugify(entry['title']), ft)
prefered_name = "{}-{}.{}".format(fn, slugify(entry['title']), ft)
grab_media(post_dir, item['href'], prefered_name) grab_media(post_dir, item['href'], prefered_name)
frontmatter['featured_image'] = prefered_name
if "summary" in entry: if "summary" in entry:
summary = entry.summary summary = entry.summary
@ -336,18 +333,6 @@ def create_opds_post(post_dir, entry):
timestamp = arrow.get(entry['updated_parsed']) timestamp = arrow.get(entry['updated_parsed'])
f.write(timestamp.format('X')) f.write(timestamp.format('X'))
def opds_fetch_more(data):
"""
Look for mode OPDS feeds to pull, untill we no longer hit the "next" navigation property.
"""
for link in data.feed.links:
for i in link:
if link[i] == 'next':
print(link['href'])
data = grab_feed(link['href'])
return data
return None
def main(): def main():
feed_urls = open("feeds_list.txt", "r").read().splitlines() feed_urls = open("feeds_list.txt", "r").read().splitlines()
@ -394,23 +379,12 @@ def main():
if data: #whenever we get a 200 if data: #whenever we get a 200
if data.feed: #only if it is an actual feed if data.feed: #only if it is an actual feed
opds_feed = False opds_feed = False
opds_entries = []
if 'links' in data.feed: if 'links' in data.feed:
for i in data.feed['links']: for i in data.feed['links']:
if i['rel'] == 'self': if i['rel'] == 'self':
if 'opds' in i['type']: if 'opds' in i['type']:
opds_feed = True opds_feed = True
print("OPDS type feed!") print("OPDS type feed!")
feed_data = data
while feed_data:
feed_data = opds_fetch_more(feed_data)
if feed_data:
for i in feed_data.entries:
opds_entries.append(i)
for i in opds_entries:
data['entries'].append(i)
for entry in data.entries: for entry in data.entries:
# if 'tags' in entry: # if 'tags' in entry: