r/konfluks
0
0
Fork 0
forked from ruangrupa/konfluks
Code Issues Pull Requests Projects Releases Wiki Activity

deps and autoformat

Browse Source
This commit is contained in:
cellarspoon
2021-12-15 11:41:35 +01:00
parent 95964862b7
commit be3e14ac6c
16 changed files with 610 additions and 387 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
lumbung-feed-aggregator/README.md
View File

@ -1,11 +0,0 @@
# lumbung feed aggregator
* Grab feeds listed in `feeds_list.txt`
* Parse feed for blogpost entries
* * Download images linked in blogposts
* Turn blogpost entries into HUGO posts
# TODO/FIXME
* only include posts with a certain tag

4
lumbung-feed-aggregator/post_template.md
View File

@ -7,7 +7,7 @@ author: "{{ frontmatter.author }}"
original_link: "{{ frontmatter.original_link }}"
feed_name: "{{ frontmatter.feed_name}}"
categories: ["network", "{{ frontmatter.feed_name}}"]
tags: {{ frontmatter.tags }}
tags: { { frontmatter.tags } }
---
{{ content }}
{{ content }}

176
lumbung-feed-aggregator/rss_aggregator.py
View File

@ -1,84 +1,88 @@
#!/bin/python3
#lumbung.space rss feed aggregator
#© 2021 roel roscam abbing gplv3 etc
# lumbung.space rss feed aggregator
# © 2021 roel roscam abbing gplv3 etc
import requests
import jinja2
import os
import shutil
import feedparser
from urllib.parse import urlparse
from ast import literal_eval as make_tuple
from slugify import slugify
from bs4 import BeautifulSoup
import time
from ast import literal_eval as make_tuple
from urllib.parse import urlparse
import arrow
import feedparser
import jinja2
import requests
from bs4 import BeautifulSoup
from slugify import slugify
def write_etag(feed_name, feed_data):
"""
save timestamp of when feed was last modified
"""
etag = ''
modified = ''
if 'etag' in feed_data:
etag = ""
modified = ""
if "etag" in feed_data:
etag = feed_data.etag
if 'modified' in feed_data:
if "modified" in feed_data:
modified = feed_data.modified
if etag or modified:
with open(os.path.join('etags',feed_name +'.txt'),'w') as f:
with open(os.path.join("etags", feed_name + ".txt"), "w") as f:
f.write(str((etag, modified)))
def get_etag(feed_name):
"""
return timestamp of when feed was last modified
"""
fn = os.path.join('etags',feed_name +'.txt')
etag = ''
modified = ''
fn = os.path.join("etags", feed_name + ".txt")
etag = ""
modified = ""
if os.path.exists(fn):
etag, modified = make_tuple(open(fn,'r').read())
etag, modified = make_tuple(open(fn, "r").read())
return etag, modified
def create_frontmatter(entry):
"""
parse RSS metadata and return as frontmatter
parse RSS metadata and return as frontmatter
"""
if 'published' in entry:
if "published" in entry:
published = entry.published_parsed
if 'updated' in entry:
if "updated" in entry:
published = entry.updated_parsed
published = arrow.get(published)
if 'author' in entry:
if "author" in entry:
author = entry.author
else:
author = ''
author = ""
tags = []
if 'tags' in entry:
#TODO finish categories
if "tags" in entry:
# TODO finish categories
for t in entry.tags:
tags.append(t['term'])
tags.append(t["term"])
frontmatter = {
'title':entry.title,
'date': published.format(),
'summary': '',
'author': author,
'original_link': entry.link,
'feed_name': entry['feed_name'],
'tags': str(tags)
"title": entry.title,
"date": published.format(),
"summary": "",
"author": author,
"original_link": entry.link,
"feed_name": entry["feed_name"],
"tags": str(tags),
}
return frontmatter
def create_post(post_dir, entry):
"""
write hugo post based on RSS entry
@ -88,40 +92,41 @@ def create_post(post_dir, entry):
if not os.path.exists(post_dir):
os.makedirs(post_dir)
if 'content' in entry:
if "content" in entry:
post_content = entry.content[0].value
else:
post_content = entry.summary
parsed_content = parse_posts(post_dir, post_content)
with open(os.path.join(post_dir,'index.html'),'w') as f: #n.b. .html
with open(os.path.join(post_dir, "index.html"), "w") as f: # n.b. .html
post = template.render(frontmatter=frontmatter, content=parsed_content)
f.write(post)
print('created post for', entry.title, '({})'.format(entry.link))
print("created post for", entry.title, "({})".format(entry.link))
def grab_media(post_directory, url):
"""
download media linked in post to have local copy
if download succeeds return new local path otherwise return url
"""
image = urlparse(url).path.split('/')[-1]
image = urlparse(url).path.split("/")[-1]
try:
if not os.path.exists(os.path.join(post_directory, image)):
#TODO: stream is true is a conditional so we could check the headers for things, mimetype etc
# TODO: stream is true is a conditional so we could check the headers for things, mimetype etc
response = requests.get(url, stream=True)
if response.ok:
with open(os.path.join(post_directory, image), 'wb') as img_file:
with open(os.path.join(post_directory, image), "wb") as img_file:
shutil.copyfileobj(response.raw, img_file)
print('Downloaded cover image', image)
print("Downloaded cover image", image)
return image
return image
elif os.path.exists(os.path.join(post_directory, image)):
return image
except Exception as e:
print('Failed to download image', url)
print("Failed to download image", url)
print(e)
return url
@ -133,27 +138,28 @@ def parse_posts(post_dir, post_content):
filter out iframe sources not in allowlist
"""
soup = BeautifulSoup(post_content, "html.parser")
allowed_iframe_sources = ['youtube.com', 'vimeo.com', 'tv.lumbung.space']
allowed_iframe_sources = ["youtube.com", "vimeo.com", "tv.lumbung.space"]
media = []
for img in soup(['img','object']):
local_image = grab_media(post_dir, img['src'])
if img['src'] != local_image:
img['src'] = local_image
for img in soup(["img", "object"]):
local_image = grab_media(post_dir, img["src"])
if img["src"] != local_image:
img["src"] = local_image
for iframe in soup(['iframe']):
if not any(source in iframe['src'] for source in allowed_iframe_sources):
print('filtered iframe: {}...'.format(iframe['src'][:25]))
for iframe in soup(["iframe"]):
if not any(source in iframe["src"] for source in allowed_iframe_sources):
print("filtered iframe: {}...".format(iframe["src"][:25]))
iframe.decompose()
return soup.decode()
def grab_feed(feed_url):
"""
check whether feed has been updated
download & return it if it has
download & return it if it has
"""
feed_name = urlparse(feed_url).netloc
etag, modified = get_etag(feed_name)
try:
@ -164,42 +170,42 @@ def grab_feed(feed_url):
else:
data = feedparser.parse(feed_url)
except Exception as e:
print('Error grabbing feed')
print("Error grabbing feed")
print(feed_name)
print(e)
return False
print(data.status, feed_url)
if data.status == 200:
#304 means the feed has not been modified since we last checked
# 304 means the feed has not been modified since we last checked
write_etag(feed_name, data)
return data
return False
feed_urls = open('feeds_list.txt','r').read().splitlines()
feed_urls = open("feeds_list.txt", "r").read().splitlines()
start = time.time()
if not os.path.exists('etags'):
os.mkdir('etags')
if not os.path.exists("etags"):
os.mkdir("etags")
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(os.path.curdir)
)
env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.curdir))
output_dir = os.environ.get('OUTPUT_DIR', '/home/r/Programming/lumbung.space/lumbung.space-web/content/posts/')
#output_dir = os.environ.get('OUTPUT_DIR', 'network/')
output_dir = os.environ.get(
"OUTPUT_DIR", "/home/r/Programming/lumbung.space/lumbung.space-web/content/posts/"
)
# output_dir = os.environ.get('OUTPUT_DIR', 'network/')
if not os.path.exists(output_dir):
os.makedirs(output_dir)
template = env.get_template('post_template.md')
template = env.get_template("post_template.md")
#add iframe to the allowlist of feedparser's sanitizer,
#this is now handled in parse_post()
feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {'iframe'}
# add iframe to the allowlist of feedparser's sanitizer,
# this is now handled in parse_post()
feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {"iframe"}
for feed_url in feed_urls:
@ -216,33 +222,33 @@ for feed_url in feed_urls:
if data:
for entry in data.entries:
# if 'tags' in entry:
# for tag in entry.tags:
# for x in ['lumbung.space', 'D15', 'lumbung']:
# if x in tag['term']:
# print(entry.title)
entry['feed_name'] = feed_name
# if 'tags' in entry:
# for tag in entry.tags:
# for x in ['lumbung.space', 'D15', 'lumbung']:
# if x in tag['term']:
# print(entry.title)
entry["feed_name"] = feed_name
post_name = slugify(entry.title)
post_dir = os.path.join(output_dir, feed_name, post_name)
if post_name not in existing_posts:
#if there is a blog entry we dont already have, make it
if post_name not in existing_posts:
# if there is a blog entry we dont already have, make it
create_post(post_dir, entry)
elif post_name in existing_posts:
#if we already have it, update it
create_post(post_dir, entry)
existing_posts.remove(post_name) # create list of posts which have not been returned by the feed
elif post_name in existing_posts:
# if we already have it, update it
create_post(post_dir, entry)
existing_posts.remove(
post_name
) # create list of posts which have not been returned by the feed
for post in existing_posts:
#remove blog posts no longer returned by the RSS feed
print('deleted', post)
for post in existing_posts:
# remove blog posts no longer returned by the RSS feed
print("deleted", post)
shutil.rmtree(os.path.join(feed_dir, slugify(post)))
end = time.time()
print(end - start)