got httpclient working, spoke heartbeat is working

This commit is contained in:
forest 2021-01-04 13:32:52 -06:00
parent d9c30e1ef8
commit 6764c5c97d
15 changed files with 281 additions and 112 deletions

View File

@ -30,6 +30,7 @@ requests = "*"
python-dotenv = "*" python-dotenv = "*"
ecdsa = "*" ecdsa = "*"
aiohttp = "*" aiohttp = "*"
apscheduler = "*"
[dev-packages] [dev-packages]

88
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "c05ae73ec64f0f248d1406a399de1ed83b7d472cf54ff0743f35d125b6d1e98f" "sha256": "b3a8d161c35cb90f0909c0531e41d0ff4598e10dfecd1e8f1ac74a3c5d6ef170"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -59,6 +59,14 @@
"index": "pypi", "index": "pypi",
"version": "==3.7.3" "version": "==3.7.3"
}, },
"apscheduler": {
"hashes": [
"sha256:3bb5229eed6fbbdafc13ce962712ae66e175aa214c69bed35a06bffcf0c5e244",
"sha256:e8b1ecdb4c7cb2818913f766d5898183c7cb8936680710a4d3a966e02262e526"
],
"index": "pypi",
"version": "==3.6.3"
},
"astroid": { "astroid": {
"hashes": [ "hashes": [
"sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1", "sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1",
@ -412,36 +420,36 @@
}, },
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a", "sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
"sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae", "sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
"sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce", "sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
"sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e", "sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
"sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140", "sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
"sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb", "sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
"sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021", "sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
"sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6", "sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
"sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302", "sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
"sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c", "sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
"sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271", "sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
"sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09", "sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
"sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3", "sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
"sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015", "sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
"sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3", "sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
"sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544", "sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
"sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8", "sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
"sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792", "sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
"sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0", "sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
"sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3", "sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
"sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8", "sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
"sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11", "sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
"sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7", "sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
"sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11", "sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
"sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e", "sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
"sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039", "sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
"sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5", "sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
"sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72" "sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d"
], ],
"version": "==8.0.1" "version": "==8.1.0"
}, },
"psycopg2": { "psycopg2": {
"hashes": [ "hashes": [
@ -492,14 +500,21 @@
"index": "pypi", "index": "pypi",
"version": "==0.15.0" "version": "==0.15.0"
}, },
"pytz": {
"hashes": [
"sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
"sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
],
"version": "==2020.5"
},
"requests": { "requests": {
"hashes": [ "hashes": [
"sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8", "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
"sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998" "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
], ],
"index": "pypi", "index": "pypi",
"markers": "python_version >= '3.0'", "markers": "python_version >= '3.0'",
"version": "==2.25.0" "version": "==2.25.1"
}, },
"six": { "six": {
"hashes": [ "hashes": [
@ -570,6 +585,13 @@
], ],
"version": "==3.7.4.3" "version": "==3.7.4.3"
}, },
"tzlocal": {
"hashes": [
"sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
"sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4"
],
"version": "==2.1"
},
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",

View File

@ -1,8 +1,10 @@
import logging import logging
from logging.config import dictConfig as logging_dict_config from logging.config import dictConfig as logging_dict_config
import atexit
import os import os
import hashlib import hashlib
import requests
import stripe import stripe
from dotenv import load_dotenv, find_dotenv from dotenv import load_dotenv, find_dotenv
@ -11,11 +13,13 @@ from flask_mail import Mail
from flask import render_template from flask import render_template
from flask import url_for from flask import url_for
from flask import current_app from flask import current_app
from apscheduler.schedulers.background import BackgroundScheduler
from capsulflask import hub_model, spoke_model, cli from capsulflask import hub_model, spoke_model, cli
from capsulflask.btcpay import client as btcpay from capsulflask.btcpay import client as btcpay
from capsulflask.http_client import MyHTTPClient from capsulflask.http_client import MyHTTPClient
load_dotenv(find_dotenv()) load_dotenv(find_dotenv())
app = Flask(__name__) app = Flask(__name__)
@ -32,7 +36,6 @@ app.config.from_mapping(
SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="default"), SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="default"),
SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="default"), SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="default"),
HUB_TOKEN=os.environ.get("HUB_TOKEN", default="default"), HUB_TOKEN=os.environ.get("HUB_TOKEN", default="default"),
HUB_URL=os.environ.get("HUB_URL", default="https://capsul.org"),
DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"), DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"), DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
@ -56,6 +59,8 @@ app.config.from_mapping(
BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club") BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club")
) )
app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL'])
logging_dict_config({ logging_dict_config({
'version': 1, 'version': 1,
'formatters': {'default': { 'formatters': {'default': {
@ -89,6 +94,19 @@ if app.config['HUB_MODE_ENABLED']:
if app.config['HUB_MODEL'] == "capsul-flask": if app.config['HUB_MODEL'] == "capsul-flask":
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub() app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
# debug mode (flask reloader) runs two copies of the app. When running in debug mode,
# we only want to start the scheduler one time.
if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
scheduler = BackgroundScheduler()
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
scheduler.add_job(func=heartbeat_task, trigger="interval", seconds=5)
scheduler.start()
atexit.register(lambda: scheduler.shutdown())
else: else:
app.config['HUB_MODEL'] = hub_model.MockHub() app.config['HUB_MODEL'] = hub_model.MockHub()
@ -107,6 +125,8 @@ if app.config['HUB_MODE_ENABLED']:
app.add_url_rule("/", endpoint="index") app.add_url_rule("/", endpoint="index")
if app.config['SPOKE_MODE_ENABLED']: if app.config['SPOKE_MODE_ENABLED']:
if app.config['SPOKE_MODEL'] == "shell-scripts": if app.config['SPOKE_MODEL'] == "shell-scripts":

View File

@ -11,7 +11,8 @@ from flask import current_app
from psycopg2 import ProgrammingError from psycopg2 import ProgrammingError
from flask_mail import Message from flask_mail import Message
from capsulflask.db import get_model, my_exec_info_message from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.console import get_account_balance from capsulflask.console import get_account_balance
bp = Blueprint('cli', __name__) bp = Blueprint('cli', __name__)

View File

@ -15,7 +15,8 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import account_required from capsulflask.auth import account_required
from capsulflask.db import get_model, my_exec_info_message from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.payment import poll_btcpay_session from capsulflask.payment import poll_btcpay_session
from capsulflask import cli from capsulflask import cli

View File

@ -40,7 +40,7 @@ def init_app(app):
hasSchemaVersionTable = False hasSchemaVersionTable = False
actionWasTaken = False actionWasTaken = False
schemaVersion = 0 schemaVersion = 0
desiredSchemaVersion = 8 desiredSchemaVersion = 9
cursor = connection.cursor() cursor = connection.cursor()
@ -126,5 +126,4 @@ def close_db(e=None):
db_model.cursor.close() db_model.cursor.close()
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection) current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)
def my_exec_info_message(exec_info):
return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])

View File

@ -4,12 +4,9 @@
from nanoid import generate from nanoid import generate
from flask import current_app from flask import current_app
from typing import List from typing import List
from capsulflask.hub_model import HTTPResult
class OnlineHost: from capsulflask.shared import OnlineHost
def __init__(self, id: str, url: str):
self.id = id
self.url = url
class DBModel: class DBModel:
#def __init__(self, connection: Psycopg2Connection, cursor: Psycopg2Cursor): #def __init__(self, connection: Psycopg2Connection, cursor: Psycopg2Cursor):
@ -277,15 +274,19 @@ class DBModel:
# ------ HOSTS --------- # ------ HOSTS ---------
def authorized_for_host(self, id, token) -> bool: def authorized_for_host(self, id, token) -> bool:
self.cursor.execute("SELECT id FROM hosts WHERE id = %s token = %s", (id, token)) self.cursor.execute("SELECT id FROM hosts WHERE id = %s AND token = %s", (id, token))
return self.cursor.fetchone() != None return self.cursor.fetchone() != None
def host_heartbeat(self, id) -> None: def host_heartbeat(self, id) -> None:
self.cursor.execute("UPDATE hosts SET last_health_check = NOW() WHERE id = %s", (id,)) self.cursor.execute("UPDATE hosts SET last_health_check = NOW() WHERE id = %s", (id,))
self.connection.commit() self.connection.commit()
def get_all_hosts(self) -> List[OnlineHost]:
self.cursor.execute("SELECT id, https_url FROM hosts")
return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))
def get_online_hosts(self) -> List[OnlineHost]: def get_online_hosts(self) -> List[OnlineHost]:
self.cursor.execute("SELECT id, https_url FROM hosts WHERE last_health_check > NOW() - INTERVAL '10 seconds'") self.cursor.execute("SELECT id, https_url FROM hosts WHERE last_health_check > NOW() - INTERVAL '20 seconds'")
return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall())) return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))
def create_operation(self, online_hosts: List[OnlineHost], email: str, payload: str) -> int: def create_operation(self, online_hosts: List[OnlineHost], email: str, payload: str) -> int:

View File

@ -1,12 +1,14 @@
import sys import sys
import json import json
import itertools
import time
import threading
import aiohttp import aiohttp
import asyncio import asyncio
from flask import current_app from flask import current_app
from capsulflask.db import my_exec_info_message from capsulflask.shared import OnlineHost, my_exec_info_message
from capsulflask.db_model import OnlineHost
from typing import List from typing import List
class HTTPResult: class HTTPResult:
@ -16,14 +18,23 @@ class HTTPResult:
class MyHTTPClient: class MyHTTPClient:
def __init__(self, timeout_seconds = 5): def __init__(self, timeout_seconds = 5):
self.client_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout_seconds)) self.timeout_seconds = timeout_seconds
self.event_loop = asyncio.get_event_loop() self.client_session = None
def make_requests_sync(self, online_hosts: List[OnlineHost], body: str) -> List(HTTPResult):
self.event_loop.run_until_complete(self.make_requests(online_hosts=online_hosts, body=body)) def make_requests_sync(self, online_hosts: List[OnlineHost], url_suffix: str, body: str, authorization_header=None) -> List[HTTPResult]:
future = run_coroutine(self.make_requests(online_hosts=online_hosts, url_suffix=url_suffix, body=body, authorization_header=authorization_header))
return future.result()
def post_json_sync(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult: def post_json_sync(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
self.event_loop.run_until_complete(self.post_json_sync(method=method, url=url, body=body)) future = run_coroutine(self.post_json(method=method, url=url, body=body, authorization_header=authorization_header))
return future.result()
def get_client_session(self):
if not self.client_session:
self.client_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=self.timeout_seconds))
return self.client_session
async def post_json(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult: async def post_json(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
response = None response = None
@ -31,12 +42,11 @@ class MyHTTPClient:
headers = {} headers = {}
if authorization_header != None: if authorization_header != None:
headers['Authorization'] = authorization_header headers['Authorization'] = authorization_header
response = await self.client_session.request( response = await self.get_client_session().request(
method=method, method=method,
url=url, url=url,
json=body, json=body,
headers=headers, headers=headers,
auth=aiohttp.BasicAuth("hub", current_app.config['HUB_TOKEN']),
verify_ssl=True, verify_ssl=True,
) )
except: except:
@ -59,15 +69,92 @@ class MyHTTPClient:
return HTTPResult(response.status, response_body) return HTTPResult(response.status, response_body)
async def make_requests(self, online_hosts: List[OnlineHost], body: str) -> List(HTTPResult): async def make_requests(self, online_hosts: List[OnlineHost], url_suffix: str, body: str, authorization_header=None) -> List[HTTPResult]:
tasks = [] tasks = []
# append to tasks in the same order as online_hosts # append to tasks in the same order as online_hosts
for host in online_hosts: for host in online_hosts:
tasks.append( tasks.append(
self.post_json(url=host.url, body=body) self.post_json(url=f"{host.url}/{url_suffix}", body=body, authorization_header=authorization_header)
) )
# gather is like Promise.all from javascript, it returns a future which resolves to an array of results # gather is like Promise.all from javascript, it returns a future which resolves to an array of results
# in the same order as the tasks that we passed in -- which were in the same order as online_hosts # in the same order as the tasks that we passed in -- which were in the same order as online_hosts
results = await asyncio.gather(*tasks) results = await asyncio.gather(*tasks)
return results return results
# i lifted this direct from https://stackoverflow.com/a/58616001
# this is the bridge between Flask's one-thread-per-request world
# and aiohttp's event-loop based world
class EventLoopThread(threading.Thread):
loop = None
_count = itertools.count(0)
def __init__(self):
name = f"{type(self).__name__}-{next(self._count)}"
super().__init__(name=name, daemon=True)
def __repr__(self):
loop, r, c, d = self.loop, False, True, False
if loop is not None:
r, c, d = loop.is_running(), loop.is_closed(), loop.get_debug()
return (
f"<{type(self).__name__} {self.name} id={self.ident} "
f"running={r} closed={c} debug={d}>"
)
def run(self):
self.loop = loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_forever()
finally:
try:
shutdown_asyncgens = loop.shutdown_asyncgens()
except AttributeError:
pass
else:
loop.run_until_complete(shutdown_asyncgens)
loop.close()
asyncio.set_event_loop(None)
def stop(self):
loop, self.loop = self.loop, None
if loop is None:
return
loop.call_soon_threadsafe(loop.stop)
self.join()
_lock = threading.Lock()
_loop_thread = None
def get_event_loop():
global _loop_thread
if _loop_thread is None:
with _lock:
if _loop_thread is None:
_loop_thread = EventLoopThread()
_loop_thread.start()
# give the thread up to a second to produce a loop
deadline = time.time() + 1
while not _loop_thread.loop and time.time() < deadline:
time.sleep(0.001)
return _loop_thread.loop
def stop_event_loop():
global _loop_thread
with _lock:
if _loop_thread is not None:
_loop_thread.stop()
_loop_thread = None
def run_coroutine(coro):
"""Run the coroutine in the event loop running in a separate thread
Returns a Future, call Future.result() to get the output
"""
return asyncio.run_coroutine_threadsafe(coro, get_event_loop())

View File

@ -4,23 +4,46 @@ from flask import current_app
from flask import request from flask import request
from werkzeug.exceptions import abort from werkzeug.exceptions import abort
from capsulflask.db import get_model, my_exec_info_message from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message, authorized_as_hub
bp = Blueprint("hub", __name__, url_prefix="/hub") bp = Blueprint("hub", __name__, url_prefix="/hub")
def authorized_for_host(id): def authorized_for_host(id):
if request.headers.get('Authorization'):
auth_header_value = request.headers.get('Authorization').replace("Bearer ", "") auth_header_value = request.headers.get('Authorization').replace("Bearer ", "")
return get_model().authorized_for_host(id, auth_header_value) return get_model().authorized_for_host(id, auth_header_value)
return False
@bp.route("/heartbeat/<string:host_id>", methods=("POST")) @bp.route("/heartbeat-task", methods=("POST",))
def ping_all_hosts_task():
if authorized_as_hub(request.headers):
all_hosts = get_model().get_all_hosts()
current_app.logger.info(f"pinging {len(all_hosts)} hosts...")
authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}"
results = current_app.config["HTTP_CLIENT"].make_requests_sync(all_hosts, "/spoke/heartbeat", None, authorization_header=authorization_header)
for i in range(len(all_hosts)):
host = all_hosts[i]
result = results[i]
current_app.logger.info(f"response from {host.id} ({host.url}): {result.status_code} {result.body}")
if result.status_code == 200:
get_model().host_heartbeat(host.id)
return "ok"
else:
current_app.logger.info(f"/hub/heartbeat-task returned 401: invalid hub token")
return abort(401, "invalid hub token")
@bp.route("/heartbeat/<string:host_id>", methods=("POST",))
def heartbeat(host_id): def heartbeat(host_id):
if authorized_for_host(host_id): if authorized_for_host(host_id):
get_model().host_heartbeat(host_id) return "ok"
else: else:
current_app.logger.info(f"/hub/heartbeat/{host_id} returned 401: invalid token") current_app.logger.info(f"/hub/heartbeat/{host_id} returned 401: invalid token")
return abort(401, "invalid host id or token") return abort(401, "invalid host id or token")
@bp.route("/claim-operation/<int:operation_id>/<string:host_id>", methods=("POST")) @bp.route("/claim-operation/<int:operation_id>/<string:host_id>", methods=("POST",))
def claim_operation(operation_id: int, host_id: str): def claim_operation(operation_id: int, host_id: str):
if authorized_for_host(host_id): if authorized_for_host(host_id):
exists = get_model().host_operation_exists(operation_id, host_id) exists = get_model().host_operation_exists(operation_id, host_id)

View File

@ -12,33 +12,9 @@ from time import sleep
from os.path import join from os.path import join
from subprocess import run from subprocess import run
from capsulflask.db_model import OnlineHost from capsulflask.db import get_model
from capsulflask.spoke_model import validate_capsul_id
from capsulflask.db import get_model, my_exec_info_message
from capsulflask.http_client import HTTPResult from capsulflask.http_client import HTTPResult
from capsulflask.shared import VirtualizationInterface, VirtualMachine, OnlineHost, validate_capsul_id, my_exec_info_message
class VirtualMachine:
def __init__(self, id, host, ipv4=None, ipv6=None):
self.id = id
self.host = host
self.ipv4 = ipv4
self.ipv6 = ipv6
class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
pass
def get(self, id: str) -> VirtualMachine:
pass
def list_ids(self) -> list:
pass
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_public_keys: list):
pass
def destroy(self, email: str, id: str):
pass
class MockHub(VirtualizationInterface): class MockHub(VirtualizationInterface):
def capacity_avaliable(self, additional_ram_bytes): def capacity_avaliable(self, additional_ram_bytes):
@ -65,7 +41,7 @@ class CapsulFlaskHub(VirtualizationInterface):
def generic_operation(self, hosts: List[OnlineHost], payload: str, immediate_mode: bool) -> Tuple[int, List[HTTPResult]]: def generic_operation(self, hosts: List[OnlineHost], payload: str, immediate_mode: bool) -> Tuple[int, List[HTTPResult]]:
operation_id = get_model().create_operation(hosts, payload) operation_id = get_model().create_operation(hosts, payload)
authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}" authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}"
results = current_app.config["HTTP_CLIENT"].make_requests_sync(hosts, payload, authorization_header=authorization_header) results = current_app.config["HTTP_CLIENT"].make_requests_sync(hosts, "/spoke/operation", payload, authorization_header=authorization_header)
for i in range(len(hosts)): for i in range(len(hosts)):
host = hosts[i] host = hosts[i]
result = results[i] result = results[i]
@ -137,7 +113,7 @@ class CapsulFlaskHub(VirtualizationInterface):
host = get_model().host_of_capsul(id) host = get_model().host_of_capsul(id)
if host is not None: if host is not None:
payload = json.dumps(dict(type="get", id=id)) payload = json.dumps(dict(type="get", id=id))
op = await self.generic_operation([host], payload, True) op = self.generic_operation([host], payload, True)
results = op[1] results = op[1]
for result in results: for result in results:
try: try:
@ -152,7 +128,7 @@ class CapsulFlaskHub(VirtualizationInterface):
def list_ids(self) -> list: def list_ids(self) -> list:
online_hosts = get_model().get_online_hosts() online_hosts = get_model().get_online_hosts()
payload = json.dumps(dict(type="list_ids")) payload = json.dumps(dict(type="list_ids"))
op = await self.generic_operation(online_hosts, payload, False) op = self.generic_operation(online_hosts, payload, False)
operation_id = op[0] operation_id = op[0]
results = op[1] results = op[1]
to_return = [] to_return = []
@ -197,7 +173,7 @@ class CapsulFlaskHub(VirtualizationInterface):
memory_mb=memory_mb, memory_mb=memory_mb,
ssh_public_keys=ssh_public_keys, ssh_public_keys=ssh_public_keys,
)) ))
op = await self.generic_operation(online_hosts, payload, False) op = self.generic_operation(online_hosts, payload, False)
operation_id = op[0] operation_id = op[0]
results = op[1] results = op[1]
number_of_assigned = 0 number_of_assigned = 0
@ -230,7 +206,7 @@ class CapsulFlaskHub(VirtualizationInterface):
host = get_model().host_of_capsul(id) host = get_model().host_of_capsul(id)
if host is not None: if host is not None:
payload = json.dumps(dict(type="destroy", id=id)) payload = json.dumps(dict(type="destroy", id=id))
op = await self.generic_operation([host], payload, True) op = self.generic_operation([host], payload, True)
results = op[1] results = op[1]
result_json_string = "<no response from host>" result_json_string = "<no response from host>"
for result in results: for result in results:

View File

@ -20,7 +20,8 @@ from werkzeug.exceptions import abort
from capsulflask.auth import account_required from capsulflask.auth import account_required
from capsulflask.db import get_model, my_exec_info_message from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
bp = Blueprint("payment", __name__, url_prefix="/payment") bp = Blueprint("payment", __name__, url_prefix="/payment")

View File

@ -7,7 +7,7 @@ CREATE TABLE hosts (
token TEXT NOT NULL token TEXT NOT NULL
); );
INSERT INTO hosts (id, token) VALUES ('baikal', 'changeme'); INSERT INTO hosts (id, https_url, token) VALUES ('baikal', 'http://localhost:5000', 'changeme');
ALTER TABLE vms ALTER TABLE vms
ADD COLUMN host TEXT REFERENCES hosts(id) ON DELETE RESTRICT DEFAULT 'baikal'; ADD COLUMN host TEXT REFERENCES hosts(id) ON DELETE RESTRICT DEFAULT 'baikal';
@ -16,7 +16,7 @@ CREATE TABLE operations (
id SERIAL PRIMARY KEY , id SERIAL PRIMARY KEY ,
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT, email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
created TIMESTAMP NOT NULL DEFAULT NOW(), created TIMESTAMP NOT NULL DEFAULT NOW(),
payload TEXT NOT NULL, payload TEXT NOT NULL
); );
CREATE TABLE host_operation ( CREATE TABLE host_operation (

44
capsulflask/shared.py Normal file
View File

@ -0,0 +1,44 @@
import re
from flask import current_app
class OnlineHost:
def __init__(self, id: str, url: str):
self.id = id
self.url = url
class VirtualMachine:
def __init__(self, id, host, ipv4=None, ipv6=None):
self.id = id
self.host = host
self.ipv4 = ipv4
self.ipv6 = ipv6
class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
pass
def get(self, id: str) -> VirtualMachine:
pass
def list_ids(self) -> list:
pass
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_public_keys: list):
pass
def destroy(self, email: str, id: str):
pass
def validate_capsul_id(id):
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
def authorized_as_hub(headers):
if headers.get('Authorization'):
auth_header_value = headers.get('Authorization').replace("Bearer ", "")
return auth_header_value == current_app.config["HUB_TOKEN"]
return False
def my_exec_info_message(exec_info):
return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])

View File

@ -7,17 +7,13 @@ from flask import request
from flask.json import jsonify from flask.json import jsonify
from werkzeug.exceptions import abort from werkzeug.exceptions import abort
from capsulflask.db import my_exec_info_message from capsulflask.shared import my_exec_info_message, authorized_as_hub
bp = Blueprint("spoke", __name__, url_prefix="/spoke") bp = Blueprint("spoke", __name__, url_prefix="/spoke")
def authorized_as_hub(id): @bp.route("/heartbeat", methods=("POST",))
auth_header_value = request.headers.get('Authorization').replace("Bearer ", "")
return auth_header_value == current_app.config["HUB_TOKEN"]
@bp.route("/heartbeat", methods=("POST"))
def heartbeat(): def heartbeat():
if authorized_as_hub(id): if authorized_as_hub(request.headers):
url = f"{current_app.config['HUB_URL']}/hub/heartbeat/{current_app.config['SPOKE_HOST_ID']}" url = f"{current_app.config['HUB_URL']}/hub/heartbeat/{current_app.config['SPOKE_HOST_ID']}"
authorization_header = f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}" authorization_header = f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}"
result = current_app.config['HTTP_CLIENT'].post_json_sync(url, body=None, authorization_header=authorization_header) result = current_app.config['HTTP_CLIENT'].post_json_sync(url, body=None, authorization_header=authorization_header)
@ -36,9 +32,9 @@ def heartbeat():
current_app.logger.info(f"/hosts/heartbeat returned 401: invalid hub token") current_app.logger.info(f"/hosts/heartbeat returned 401: invalid hub token")
return abort(401, "invalid hub token") return abort(401, "invalid hub token")
@bp.route("/operation", methods=("POST")) @bp.route("/operation", methods=("POST",))
def operation(): def operation():
if authorized_as_hub(id): if authorized_as_hub(request.headers):
request_body = request.json() request_body = request.json()
handlers = { handlers = {
"capacity_avaliable": handle_capacity_avaliable, "capacity_avaliable": handle_capacity_avaliable,

View File

@ -8,11 +8,8 @@ from subprocess import run
from capsulflask.db import get_model from capsulflask.db import get_model
from capsulflask.hub_model import VirtualizationInterface, VirtualMachine from capsulflask.shared import VirtualizationInterface, VirtualMachine, validate_capsul_id
def validate_capsul_id(id):
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
class MockSpoke(VirtualizationInterface): class MockSpoke(VirtualizationInterface):
def capacity_avaliable(self, additional_ram_bytes): def capacity_avaliable(self, additional_ram_bytes):