feito fork de ruangrupa/konfluks
Comparar commits
1 Commits
36b4ca1ba4
...
dev
| Autor | SHA1 | Data | |
|---|---|---|---|
48e1838a2c
|
5
.gitignore
externo
5
.gitignore
externo
@ -1,7 +1,4 @@
|
||||
*.txt
|
||||
*.txt.*
|
||||
__pycache__
|
||||
etags
|
||||
test
|
||||
__pycache__
|
||||
.venv
|
||||
content
|
||||
|
||||
@ -1,8 +1,3 @@
|
||||
# lumbunglib
|
||||
|
||||
> Python lib which powers `lumbung[dot]space` automation
|
||||
|
||||
## hacking
|
||||
|
||||
- run `poetry run poetry2setup > setup.py` if updating the poetry dependencies
|
||||
- if adding a new cli command, extend `pyproject.toml` with a new `[tool.poetry.scripts]` entry
|
||||
|
||||
@ -60,11 +60,6 @@ def create_frontmatter(entry):
|
||||
else:
|
||||
author = ""
|
||||
|
||||
if "title" in entry:
|
||||
title = entry.title
|
||||
else:
|
||||
title = ""
|
||||
|
||||
tags = []
|
||||
if "tags" in entry:
|
||||
# TODO finish categories
|
||||
@ -72,7 +67,7 @@ def create_frontmatter(entry):
|
||||
tags.append(t["term"])
|
||||
|
||||
frontmatter = {
|
||||
"title": title,
|
||||
"title": entry.title,
|
||||
"date": published.format(),
|
||||
"summary": "",
|
||||
"author": author,
|
||||
@ -199,25 +194,13 @@ def main():
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
feed_dict = dict()
|
||||
for url in feed_urls:
|
||||
feed_name = urlparse(url).netloc
|
||||
feed_dict[url] = feed_name
|
||||
|
||||
feed_names = feed_dict.values()
|
||||
content_dirs = os.listdir(output_dir)
|
||||
for i in content_dirs:
|
||||
if i not in feed_names:
|
||||
shutil.rmtree(os.path.join(output_dir, i))
|
||||
print("%s not in feeds_list.txt, removing local data" %(i))
|
||||
|
||||
# add iframe to the allowlist of feedparser's sanitizer,
|
||||
# this is now handled in parse_post()
|
||||
feedparser.sanitizer._HTMLSanitizer.acceptable_elements |= {"iframe"}
|
||||
|
||||
for feed_url in feed_urls:
|
||||
|
||||
feed_name = feed_dict[feed_url]
|
||||
feed_name = urlparse(feed_url).netloc
|
||||
|
||||
feed_dir = os.path.join(output_dir, feed_name)
|
||||
|
||||
|
||||
@ -9,15 +9,7 @@ from mastodon import Mastodon
|
||||
instance = "https://social.lumbung.space"
|
||||
email = ""
|
||||
password = ""
|
||||
hashtags = [
|
||||
"documentafifteen",
|
||||
"harvestedbyputra",
|
||||
"jalansesama",
|
||||
"lumbungdotspace",
|
||||
"majelisakakbar",
|
||||
"majelisakbar",
|
||||
"warungkopi",
|
||||
]
|
||||
hashtags = ["jalansesama"]
|
||||
|
||||
|
||||
def login_mastodon_bot():
|
||||
@ -103,10 +95,6 @@ def main():
|
||||
if not os.path.exists(output_dir):
|
||||
os.mkdir(output_dir)
|
||||
|
||||
all_existing_posts = []
|
||||
for i in os.listdir(output_dir):
|
||||
all_existing_posts += os.listdir(os.path.join(output_dir, i))
|
||||
|
||||
for hashtag in hashtags:
|
||||
|
||||
hashtag_dir = os.path.join(output_dir, hashtag)
|
||||
@ -126,13 +114,12 @@ def main():
|
||||
post_dir = os.path.join(hashtag_dir, str(post_metadata["id"]))
|
||||
|
||||
# if there is a post in the feed we dont already have locally, make it
|
||||
if str(post_metadata["id"]) not in all_existing_posts:
|
||||
if str(post_metadata["id"]) not in existing_posts:
|
||||
|
||||
if not post_metadata[
|
||||
"local_only"
|
||||
]: # if you get an error here then you are using vanilla Mastodon, this is a Hometown or Glitch only feature
|
||||
create_post(post_dir, post_metadata)
|
||||
all_existing_posts.append(post_metadata["id"])
|
||||
else:
|
||||
print("not pulling post %s (post is local only)" % (post_metadata["id"]))
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ author: "{{ frontmatter.author }}"
|
||||
original_link: "{{ frontmatter.original_link }}"
|
||||
feed_name: "{{ frontmatter.feed_name}}"
|
||||
categories: ["network", "{{ frontmatter.feed_name}}"]
|
||||
tags: {{ frontmatter.tags }}
|
||||
tags: { { frontmatter.tags } }
|
||||
---
|
||||
|
||||
{{ content }}
|
||||
|
||||
73
poetry.lock
gerado
73
poetry.lock
gerado
@ -9,21 +9,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
[package.dependencies]
|
||||
python-dateutil = "*"
|
||||
|
||||
[[package]]
|
||||
name = "beautifulsoup4"
|
||||
version = "4.10.0"
|
||||
description = "Screen-scraping library"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">3.0.0"
|
||||
|
||||
[package.dependencies]
|
||||
soupsieve = ">1.2"
|
||||
|
||||
[package.extras]
|
||||
html5lib = ["html5lib"]
|
||||
lxml = ["lxml"]
|
||||
|
||||
[[package]]
|
||||
name = "blurhash"
|
||||
version = "1.1.4"
|
||||
@ -35,17 +20,6 @@ python-versions = "*"
|
||||
[package.extras]
|
||||
test = ["pillow", "numpy", "pytest"]
|
||||
|
||||
[[package]]
|
||||
name = "bs4"
|
||||
version = "0.0.1"
|
||||
description = "Dummy package for Beautiful Soup"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[package.dependencies]
|
||||
beautifulsoup4 = "*"
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2021.10.8"
|
||||
@ -73,17 +47,6 @@ category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[[package]]
|
||||
name = "feedparser"
|
||||
version = "6.0.8"
|
||||
description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[package.dependencies]
|
||||
sgmllib3k = "*"
|
||||
|
||||
[[package]]
|
||||
name = "ics"
|
||||
version = "0.7"
|
||||
@ -260,14 +223,6 @@ urllib3 = ">=1.21.1,<1.27"
|
||||
socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
|
||||
use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
|
||||
|
||||
[[package]]
|
||||
name = "sgmllib3k"
|
||||
version = "1.0.0"
|
||||
description = "Py3k port of sgmllib."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "six"
|
||||
version = "1.16.0"
|
||||
@ -276,14 +231,6 @@ category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.3.1"
|
||||
description = "A modern CSS selector implementation for Beautiful Soup."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[[package]]
|
||||
name = "tatsu"
|
||||
version = "5.7.0"
|
||||
@ -319,24 +266,17 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "c5c987253f949737210f4a3d3c3c24b0affd4a9c7d06de386c9bd514c592db8b"
|
||||
content-hash = "d3c9d528613826932cb3f316a3a69066e87e69011d4c8d3a2492521833d3851f"
|
||||
|
||||
[metadata.files]
|
||||
arrow = [
|
||||
{file = "arrow-0.14.7-py2.py3-none-any.whl", hash = "sha256:4bfacea734ead51495dc47df00421ecfd4ca1f2c0fbe58b9a26eaeddedc31caf"},
|
||||
{file = "arrow-0.14.7.tar.gz", hash = "sha256:67f8be7c0cf420424bc62d8d7dc40b44e4bb2f7b515f9cc2954fb36e35797656"},
|
||||
]
|
||||
beautifulsoup4 = [
|
||||
{file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"},
|
||||
{file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"},
|
||||
]
|
||||
blurhash = [
|
||||
{file = "blurhash-1.1.4-py2.py3-none-any.whl", hash = "sha256:7611c1bc41383d2349b6129208587b5d61e8792ce953893cb49c38beeb400d1d"},
|
||||
{file = "blurhash-1.1.4.tar.gz", hash = "sha256:da56b163e5a816e4ad07172f5639287698e09d7f3dc38d18d9726d9c1dbc4cee"},
|
||||
]
|
||||
bs4 = [
|
||||
{file = "bs4-0.0.1.tar.gz", hash = "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"},
|
||||
]
|
||||
certifi = [
|
||||
{file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
|
||||
{file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
|
||||
@ -349,10 +289,6 @@ decorator = [
|
||||
{file = "decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186"},
|
||||
{file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
|
||||
]
|
||||
feedparser = [
|
||||
{file = "feedparser-6.0.8-py3-none-any.whl", hash = "sha256:1b7f57841d9cf85074deb316ed2c795091a238adb79846bc46dccdaf80f9c59a"},
|
||||
{file = "feedparser-6.0.8.tar.gz", hash = "sha256:5ce0410a05ab248c8c7cfca3a0ea2203968ee9ff4486067379af4827a59f9661"},
|
||||
]
|
||||
ics = [
|
||||
{file = "ics-0.7-py2.py3-none-any.whl", hash = "sha256:bf5fbdef6e1e073afdadf1b996f0271186dd114a148e38e795919a1ae644d6ac"},
|
||||
{file = "ics-0.7-py3.7.egg", hash = "sha256:3b606205b9582ad27dff77f9b227a30d02fdac532731927fe39df1f1ddf8673f"},
|
||||
@ -473,17 +409,10 @@ requests = [
|
||||
{file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
|
||||
{file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
|
||||
]
|
||||
sgmllib3k = [
|
||||
{file = "sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9"},
|
||||
]
|
||||
six = [
|
||||
{file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
|
||||
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
|
||||
]
|
||||
soupsieve = [
|
||||
{file = "soupsieve-2.3.1-py3-none-any.whl", hash = "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb"},
|
||||
{file = "soupsieve-2.3.1.tar.gz", hash = "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"},
|
||||
]
|
||||
tatsu = [
|
||||
{file = "TatSu-5.7.0-py2.py3-none-any.whl", hash = "sha256:9eebadfc2889d8e82e197df22913df56ff204bf4cfc62db49a5c7edd084e10b4"},
|
||||
{file = "TatSu-5.7.0.zip", hash = "sha256:428136cd4aa9600fcd01428bd5667fc752062f54bd0148dc1e64fee7b8d05fa4"},
|
||||
|
||||
@ -13,8 +13,6 @@ natural = "^0.2.0"
|
||||
python-slugify = "^5.0.2"
|
||||
requests = "^2.26.0"
|
||||
peertube = {git = "https://framagit.org/framasoft/peertube/clients/python.git"}
|
||||
feedparser = "^6.0.8"
|
||||
bs4 = "^0.0.1"
|
||||
"Mastodon.py" = "^1.5.1"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
@ -27,5 +25,4 @@ build-backend = "poetry.core.masonry.api"
|
||||
[tool.poetry.scripts]
|
||||
lumbunglib-cal = "lumbunglib.cloudcal:main"
|
||||
lumbunglib-vid = "lumbunglib.video:main"
|
||||
lumbunglib-feed = "lumbunglib.feed:main"
|
||||
lumbunglib-hash = "lumbunglib.hashtag:main"
|
||||
|
||||
3
setup.py
3
setup.py
@ -10,8 +10,6 @@ package_data = \
|
||||
install_requires = \
|
||||
['Jinja2>=3.0.3,<4.0.0',
|
||||
'Mastodon.py>=1.5.1,<2.0.0',
|
||||
'bs4>=0.0.1,<0.0.2',
|
||||
'feedparser>=6.0.8,<7.0.0',
|
||||
'ics>=0.7,<0.8',
|
||||
'natural>=0.2.0,<0.3.0',
|
||||
'peertube @ '
|
||||
@ -21,7 +19,6 @@ install_requires = \
|
||||
|
||||
entry_points = \
|
||||
{'console_scripts': ['lumbunglib-cal = lumbunglib.cloudcal:main',
|
||||
'lumbunglib-feed = lumbunglib.feed:main',
|
||||
'lumbunglib-hash = lumbunglib.hashtag:main',
|
||||
'lumbunglib-vid = lumbunglib.video:main']}
|
||||
|
||||
|
||||
Referência em uma nova issue
Block a user