got httpclient working, spoke heartbeat is working
This commit is contained in:
parent
d9c30e1ef8
commit
6764c5c97d
1
Pipfile
1
Pipfile
@ -30,6 +30,7 @@ requests = "*"
|
|||||||
python-dotenv = "*"
|
python-dotenv = "*"
|
||||||
ecdsa = "*"
|
ecdsa = "*"
|
||||||
aiohttp = "*"
|
aiohttp = "*"
|
||||||
|
apscheduler = "*"
|
||||||
|
|
||||||
[dev-packages]
|
[dev-packages]
|
||||||
|
|
||||||
|
88
Pipfile.lock
generated
88
Pipfile.lock
generated
@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"_meta": {
|
"_meta": {
|
||||||
"hash": {
|
"hash": {
|
||||||
"sha256": "c05ae73ec64f0f248d1406a399de1ed83b7d472cf54ff0743f35d125b6d1e98f"
|
"sha256": "b3a8d161c35cb90f0909c0531e41d0ff4598e10dfecd1e8f1ac74a3c5d6ef170"
|
||||||
},
|
},
|
||||||
"pipfile-spec": 6,
|
"pipfile-spec": 6,
|
||||||
"requires": {
|
"requires": {
|
||||||
@ -59,6 +59,14 @@
|
|||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==3.7.3"
|
"version": "==3.7.3"
|
||||||
},
|
},
|
||||||
|
"apscheduler": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:3bb5229eed6fbbdafc13ce962712ae66e175aa214c69bed35a06bffcf0c5e244",
|
||||||
|
"sha256:e8b1ecdb4c7cb2818913f766d5898183c7cb8936680710a4d3a966e02262e526"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==3.6.3"
|
||||||
|
},
|
||||||
"astroid": {
|
"astroid": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1",
|
"sha256:4c17cea3e592c21b6e222f673868961bad77e1f985cb1694ed077475a89229c1",
|
||||||
@ -412,36 +420,36 @@
|
|||||||
},
|
},
|
||||||
"pillow": {
|
"pillow": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:006de60d7580d81f4a1a7e9f0173dc90a932e3905cc4d47ea909bc946302311a",
|
"sha256:165c88bc9d8dba670110c689e3cc5c71dbe4bfb984ffa7cbebf1fac9554071d6",
|
||||||
"sha256:0a2e8d03787ec7ad71dc18aec9367c946ef8ef50e1e78c71f743bc3a770f9fae",
|
"sha256:22d070ca2e60c99929ef274cfced04294d2368193e935c5d6febfd8b601bf865",
|
||||||
"sha256:0eeeae397e5a79dc088d8297a4c2c6f901f8fb30db47795113a4a605d0f1e5ce",
|
"sha256:2353834b2c49b95e1313fb34edf18fca4d57446675d05298bb694bca4b194174",
|
||||||
"sha256:11c5c6e9b02c9dac08af04f093eb5a2f84857df70a7d4a6a6ad461aca803fb9e",
|
"sha256:39725acf2d2e9c17356e6835dccebe7a697db55f25a09207e38b835d5e1bc032",
|
||||||
"sha256:2fb113757a369a6cdb189f8df3226e995acfed0a8919a72416626af1a0a71140",
|
"sha256:3de6b2ee4f78c6b3d89d184ade5d8fa68af0848f9b6b6da2b9ab7943ec46971a",
|
||||||
"sha256:4b0ef2470c4979e345e4e0cc1bbac65fda11d0d7b789dbac035e4c6ce3f98adb",
|
"sha256:47c0d93ee9c8b181f353dbead6530b26980fe4f5485aa18be8f1fd3c3cbc685e",
|
||||||
"sha256:59e903ca800c8cfd1ebe482349ec7c35687b95e98cefae213e271c8c7fffa021",
|
"sha256:5e2fe3bb2363b862671eba632537cd3a823847db4d98be95690b7e382f3d6378",
|
||||||
"sha256:5abd653a23c35d980b332bc0431d39663b1709d64142e3652890df4c9b6970f6",
|
"sha256:604815c55fd92e735f9738f65dabf4edc3e79f88541c221d292faec1904a4b17",
|
||||||
"sha256:5f9403af9c790cc18411ea398a6950ee2def2a830ad0cfe6dc9122e6d528b302",
|
"sha256:6c5275bd82711cd3dcd0af8ce0bb99113ae8911fc2952805f1d012de7d600a4c",
|
||||||
"sha256:6b4a8fd632b4ebee28282a9fef4c341835a1aa8671e2770b6f89adc8e8c2703c",
|
"sha256:731ca5aabe9085160cf68b2dbef95fc1991015bc0a3a6ea46a371ab88f3d0913",
|
||||||
"sha256:6c1aca8231625115104a06e4389fcd9ec88f0c9befbabd80dc206c35561be271",
|
"sha256:7612520e5e1a371d77e1d1ca3a3ee6227eef00d0a9cddb4ef7ecb0b7396eddf7",
|
||||||
"sha256:795e91a60f291e75de2e20e6bdd67770f793c8605b553cb6e4387ce0cb302e09",
|
"sha256:7916cbc94f1c6b1301ac04510d0881b9e9feb20ae34094d3615a8a7c3db0dcc0",
|
||||||
"sha256:7ba0ba61252ab23052e642abdb17fd08fdcfdbbf3b74c969a30c58ac1ade7cd3",
|
"sha256:81c3fa9a75d9f1afafdb916d5995633f319db09bd773cb56b8e39f1e98d90820",
|
||||||
"sha256:7c9401e68730d6c4245b8e361d3d13e1035cbc94db86b49dc7da8bec235d0015",
|
"sha256:887668e792b7edbfb1d3c9d8b5d8c859269a0f0eba4dda562adb95500f60dbba",
|
||||||
"sha256:81f812d8f5e8a09b246515fac141e9d10113229bc33ea073fec11403b016bcf3",
|
"sha256:93a473b53cc6e0b3ce6bf51b1b95b7b1e7e6084be3a07e40f79b42e83503fbf2",
|
||||||
"sha256:895d54c0ddc78a478c80f9c438579ac15f3e27bf442c2a9aa74d41d0e4d12544",
|
"sha256:96d4dc103d1a0fa6d47c6c55a47de5f5dafd5ef0114fa10c85a1fd8e0216284b",
|
||||||
"sha256:8de332053707c80963b589b22f8e0229f1be1f3ca862a932c1bcd48dafb18dd8",
|
"sha256:a3d3e086474ef12ef13d42e5f9b7bbf09d39cf6bd4940f982263d6954b13f6a9",
|
||||||
"sha256:92c882b70a40c79de9f5294dc99390671e07fc0b0113d472cbea3fde15db1792",
|
"sha256:b02a0b9f332086657852b1f7cb380f6a42403a6d9c42a4c34a561aa4530d5234",
|
||||||
"sha256:95edb1ed513e68bddc2aee3de66ceaf743590bf16c023fb9977adc4be15bd3f0",
|
"sha256:b09e10ec453de97f9a23a5aa5e30b334195e8d2ddd1ce76cc32e52ba63c8b31d",
|
||||||
"sha256:b63d4ff734263ae4ce6593798bcfee6dbfb00523c82753a3a03cbc05555a9cc3",
|
"sha256:b6f00ad5ebe846cc91763b1d0c6d30a8042e02b2316e27b05de04fa6ec831ec5",
|
||||||
"sha256:bd7bf289e05470b1bc74889d1466d9ad4a56d201f24397557b6f65c24a6844b8",
|
"sha256:bba80df38cfc17f490ec651c73bb37cd896bc2400cfba27d078c2135223c1206",
|
||||||
"sha256:cc3ea6b23954da84dbee8025c616040d9aa5eaf34ea6895a0a762ee9d3e12e11",
|
"sha256:c3d911614b008e8a576b8e5303e3db29224b455d3d66d1b2848ba6ca83f9ece9",
|
||||||
"sha256:cc9ec588c6ef3a1325fa032ec14d97b7309db493782ea8c304666fb10c3bd9a7",
|
"sha256:ca20739e303254287138234485579b28cb0d524401f83d5129b5ff9d606cb0a8",
|
||||||
"sha256:d3d07c86d4efa1facdf32aa878bd508c0dc4f87c48125cc16b937baa4e5b5e11",
|
"sha256:cb192176b477d49b0a327b2a5a4979552b7a58cd42037034316b8018ac3ebb59",
|
||||||
"sha256:d8a96747df78cda35980905bf26e72960cba6d355ace4780d4bdde3b217cdf1e",
|
"sha256:cdbbe7dff4a677fb555a54f9bc0450f2a21a93c5ba2b44e09e54fcb72d2bd13d",
|
||||||
"sha256:e38d58d9138ef972fceb7aeec4be02e3f01d383723965bfcef14d174c8ccd039",
|
"sha256:d355502dce85ade85a2511b40b4c61a128902f246504f7de29bbeec1ae27933a",
|
||||||
"sha256:eb472586374dc66b31e36e14720747595c2b265ae962987261f044e5cce644b5",
|
"sha256:dc577f4cfdda354db3ae37a572428a90ffdbe4e51eda7849bf442fb803f09c9b",
|
||||||
"sha256:fbd922f702582cb0d71ef94442bfca57624352622d75e3be7a1e7e9360b07e72"
|
"sha256:dd9eef866c70d2cbbea1ae58134eaffda0d4bfea403025f4db6859724b18ab3d"
|
||||||
],
|
],
|
||||||
"version": "==8.0.1"
|
"version": "==8.1.0"
|
||||||
},
|
},
|
||||||
"psycopg2": {
|
"psycopg2": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@ -492,14 +500,21 @@
|
|||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"version": "==0.15.0"
|
"version": "==0.15.0"
|
||||||
},
|
},
|
||||||
|
"pytz": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",
|
||||||
|
"sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"
|
||||||
|
],
|
||||||
|
"version": "==2020.5"
|
||||||
|
},
|
||||||
"requests": {
|
"requests": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:7f1a0b932f4a60a1a65caa4263921bb7d9ee911957e0ae4a23a6dd08185ad5f8",
|
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
|
||||||
"sha256:e786fa28d8c9154e6a4de5d46a1d921b8749f8b74e28bde23768e5e16eece998"
|
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
|
||||||
],
|
],
|
||||||
"index": "pypi",
|
"index": "pypi",
|
||||||
"markers": "python_version >= '3.0'",
|
"markers": "python_version >= '3.0'",
|
||||||
"version": "==2.25.0"
|
"version": "==2.25.1"
|
||||||
},
|
},
|
||||||
"six": {
|
"six": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
@ -570,6 +585,13 @@
|
|||||||
],
|
],
|
||||||
"version": "==3.7.4.3"
|
"version": "==3.7.4.3"
|
||||||
},
|
},
|
||||||
|
"tzlocal": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44",
|
||||||
|
"sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4"
|
||||||
|
],
|
||||||
|
"version": "==2.1"
|
||||||
|
},
|
||||||
"urllib3": {
|
"urllib3": {
|
||||||
"hashes": [
|
"hashes": [
|
||||||
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
|
"sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",
|
||||||
|
@ -1,8 +1,10 @@
|
|||||||
import logging
|
import logging
|
||||||
from logging.config import dictConfig as logging_dict_config
|
from logging.config import dictConfig as logging_dict_config
|
||||||
|
|
||||||
|
import atexit
|
||||||
import os
|
import os
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import requests
|
||||||
|
|
||||||
import stripe
|
import stripe
|
||||||
from dotenv import load_dotenv, find_dotenv
|
from dotenv import load_dotenv, find_dotenv
|
||||||
@ -11,11 +13,13 @@ from flask_mail import Mail
|
|||||||
from flask import render_template
|
from flask import render_template
|
||||||
from flask import url_for
|
from flask import url_for
|
||||||
from flask import current_app
|
from flask import current_app
|
||||||
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
|
|
||||||
from capsulflask import hub_model, spoke_model, cli
|
from capsulflask import hub_model, spoke_model, cli
|
||||||
from capsulflask.btcpay import client as btcpay
|
from capsulflask.btcpay import client as btcpay
|
||||||
from capsulflask.http_client import MyHTTPClient
|
from capsulflask.http_client import MyHTTPClient
|
||||||
|
|
||||||
|
|
||||||
load_dotenv(find_dotenv())
|
load_dotenv(find_dotenv())
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
@ -32,7 +36,6 @@ app.config.from_mapping(
|
|||||||
SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="default"),
|
SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="default"),
|
||||||
SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="default"),
|
SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="default"),
|
||||||
HUB_TOKEN=os.environ.get("HUB_TOKEN", default="default"),
|
HUB_TOKEN=os.environ.get("HUB_TOKEN", default="default"),
|
||||||
HUB_URL=os.environ.get("HUB_URL", default="https://capsul.org"),
|
|
||||||
|
|
||||||
DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
|
DATABASE_URL=os.environ.get("DATABASE_URL", default="sql://postgres:dev@localhost:5432/postgres"),
|
||||||
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
|
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
|
||||||
@ -56,6 +59,8 @@ app.config.from_mapping(
|
|||||||
BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club")
|
BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL'])
|
||||||
|
|
||||||
logging_dict_config({
|
logging_dict_config({
|
||||||
'version': 1,
|
'version': 1,
|
||||||
'formatters': {'default': {
|
'formatters': {'default': {
|
||||||
@ -89,6 +94,19 @@ if app.config['HUB_MODE_ENABLED']:
|
|||||||
|
|
||||||
if app.config['HUB_MODEL'] == "capsul-flask":
|
if app.config['HUB_MODEL'] == "capsul-flask":
|
||||||
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
|
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
|
||||||
|
|
||||||
|
# debug mode (flask reloader) runs two copies of the app. When running in debug mode,
|
||||||
|
# we only want to start the scheduler one time.
|
||||||
|
if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
|
||||||
|
scheduler = BackgroundScheduler()
|
||||||
|
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
|
||||||
|
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
|
||||||
|
heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
|
||||||
|
scheduler.add_job(func=heartbeat_task, trigger="interval", seconds=5)
|
||||||
|
scheduler.start()
|
||||||
|
|
||||||
|
atexit.register(lambda: scheduler.shutdown())
|
||||||
|
|
||||||
else:
|
else:
|
||||||
app.config['HUB_MODEL'] = hub_model.MockHub()
|
app.config['HUB_MODEL'] = hub_model.MockHub()
|
||||||
|
|
||||||
@ -107,6 +125,8 @@ if app.config['HUB_MODE_ENABLED']:
|
|||||||
|
|
||||||
app.add_url_rule("/", endpoint="index")
|
app.add_url_rule("/", endpoint="index")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if app.config['SPOKE_MODE_ENABLED']:
|
if app.config['SPOKE_MODE_ENABLED']:
|
||||||
|
|
||||||
if app.config['SPOKE_MODEL'] == "shell-scripts":
|
if app.config['SPOKE_MODEL'] == "shell-scripts":
|
||||||
|
@ -11,7 +11,8 @@ from flask import current_app
|
|||||||
from psycopg2 import ProgrammingError
|
from psycopg2 import ProgrammingError
|
||||||
from flask_mail import Message
|
from flask_mail import Message
|
||||||
|
|
||||||
from capsulflask.db import get_model, my_exec_info_message
|
from capsulflask.db import get_model
|
||||||
|
from capsulflask.shared import my_exec_info_message
|
||||||
from capsulflask.console import get_account_balance
|
from capsulflask.console import get_account_balance
|
||||||
|
|
||||||
bp = Blueprint('cli', __name__)
|
bp = Blueprint('cli', __name__)
|
||||||
|
@ -15,7 +15,8 @@ from nanoid import generate
|
|||||||
|
|
||||||
from capsulflask.metrics import durations as metric_durations
|
from capsulflask.metrics import durations as metric_durations
|
||||||
from capsulflask.auth import account_required
|
from capsulflask.auth import account_required
|
||||||
from capsulflask.db import get_model, my_exec_info_message
|
from capsulflask.db import get_model
|
||||||
|
from capsulflask.shared import my_exec_info_message
|
||||||
from capsulflask.payment import poll_btcpay_session
|
from capsulflask.payment import poll_btcpay_session
|
||||||
from capsulflask import cli
|
from capsulflask import cli
|
||||||
|
|
||||||
|
@ -40,7 +40,7 @@ def init_app(app):
|
|||||||
hasSchemaVersionTable = False
|
hasSchemaVersionTable = False
|
||||||
actionWasTaken = False
|
actionWasTaken = False
|
||||||
schemaVersion = 0
|
schemaVersion = 0
|
||||||
desiredSchemaVersion = 8
|
desiredSchemaVersion = 9
|
||||||
|
|
||||||
cursor = connection.cursor()
|
cursor = connection.cursor()
|
||||||
|
|
||||||
@ -126,5 +126,4 @@ def close_db(e=None):
|
|||||||
db_model.cursor.close()
|
db_model.cursor.close()
|
||||||
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)
|
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)
|
||||||
|
|
||||||
def my_exec_info_message(exec_info):
|
|
||||||
return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])
|
|
||||||
|
@ -4,12 +4,9 @@
|
|||||||
from nanoid import generate
|
from nanoid import generate
|
||||||
from flask import current_app
|
from flask import current_app
|
||||||
from typing import List
|
from typing import List
|
||||||
from capsulflask.hub_model import HTTPResult
|
|
||||||
|
|
||||||
class OnlineHost:
|
from capsulflask.shared import OnlineHost
|
||||||
def __init__(self, id: str, url: str):
|
|
||||||
self.id = id
|
|
||||||
self.url = url
|
|
||||||
|
|
||||||
class DBModel:
|
class DBModel:
|
||||||
#def __init__(self, connection: Psycopg2Connection, cursor: Psycopg2Cursor):
|
#def __init__(self, connection: Psycopg2Connection, cursor: Psycopg2Cursor):
|
||||||
@ -277,15 +274,19 @@ class DBModel:
|
|||||||
# ------ HOSTS ---------
|
# ------ HOSTS ---------
|
||||||
|
|
||||||
def authorized_for_host(self, id, token) -> bool:
|
def authorized_for_host(self, id, token) -> bool:
|
||||||
self.cursor.execute("SELECT id FROM hosts WHERE id = %s token = %s", (id, token))
|
self.cursor.execute("SELECT id FROM hosts WHERE id = %s AND token = %s", (id, token))
|
||||||
return self.cursor.fetchone() != None
|
return self.cursor.fetchone() != None
|
||||||
|
|
||||||
def host_heartbeat(self, id) -> None:
|
def host_heartbeat(self, id) -> None:
|
||||||
self.cursor.execute("UPDATE hosts SET last_health_check = NOW() WHERE id = %s", (id,))
|
self.cursor.execute("UPDATE hosts SET last_health_check = NOW() WHERE id = %s", (id,))
|
||||||
self.connection.commit()
|
self.connection.commit()
|
||||||
|
|
||||||
|
def get_all_hosts(self) -> List[OnlineHost]:
|
||||||
|
self.cursor.execute("SELECT id, https_url FROM hosts")
|
||||||
|
return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))
|
||||||
|
|
||||||
def get_online_hosts(self) -> List[OnlineHost]:
|
def get_online_hosts(self) -> List[OnlineHost]:
|
||||||
self.cursor.execute("SELECT id, https_url FROM hosts WHERE last_health_check > NOW() - INTERVAL '10 seconds'")
|
self.cursor.execute("SELECT id, https_url FROM hosts WHERE last_health_check > NOW() - INTERVAL '20 seconds'")
|
||||||
return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))
|
return list(map(lambda x: OnlineHost(id=x[0], url=x[1]), self.cursor.fetchall()))
|
||||||
|
|
||||||
def create_operation(self, online_hosts: List[OnlineHost], email: str, payload: str) -> int:
|
def create_operation(self, online_hosts: List[OnlineHost], email: str, payload: str) -> int:
|
||||||
|
@ -1,12 +1,14 @@
|
|||||||
|
|
||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
|
import itertools
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
from flask import current_app
|
from flask import current_app
|
||||||
from capsulflask.db import my_exec_info_message
|
from capsulflask.shared import OnlineHost, my_exec_info_message
|
||||||
from capsulflask.db_model import OnlineHost
|
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
class HTTPResult:
|
class HTTPResult:
|
||||||
@ -16,14 +18,23 @@ class HTTPResult:
|
|||||||
|
|
||||||
class MyHTTPClient:
|
class MyHTTPClient:
|
||||||
def __init__(self, timeout_seconds = 5):
|
def __init__(self, timeout_seconds = 5):
|
||||||
self.client_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout_seconds))
|
self.timeout_seconds = timeout_seconds
|
||||||
self.event_loop = asyncio.get_event_loop()
|
self.client_session = None
|
||||||
|
|
||||||
def make_requests_sync(self, online_hosts: List[OnlineHost], body: str) -> List(HTTPResult):
|
|
||||||
self.event_loop.run_until_complete(self.make_requests(online_hosts=online_hosts, body=body))
|
def make_requests_sync(self, online_hosts: List[OnlineHost], url_suffix: str, body: str, authorization_header=None) -> List[HTTPResult]:
|
||||||
|
future = run_coroutine(self.make_requests(online_hosts=online_hosts, url_suffix=url_suffix, body=body, authorization_header=authorization_header))
|
||||||
|
return future.result()
|
||||||
|
|
||||||
def post_json_sync(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
|
def post_json_sync(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
|
||||||
self.event_loop.run_until_complete(self.post_json_sync(method=method, url=url, body=body))
|
future = run_coroutine(self.post_json(method=method, url=url, body=body, authorization_header=authorization_header))
|
||||||
|
return future.result()
|
||||||
|
|
||||||
|
def get_client_session(self):
|
||||||
|
if not self.client_session:
|
||||||
|
self.client_session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=self.timeout_seconds))
|
||||||
|
|
||||||
|
return self.client_session
|
||||||
|
|
||||||
async def post_json(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
|
async def post_json(self, url: str, body: str, method="POST", authorization_header=None) -> HTTPResult:
|
||||||
response = None
|
response = None
|
||||||
@ -31,12 +42,11 @@ class MyHTTPClient:
|
|||||||
headers = {}
|
headers = {}
|
||||||
if authorization_header != None:
|
if authorization_header != None:
|
||||||
headers['Authorization'] = authorization_header
|
headers['Authorization'] = authorization_header
|
||||||
response = await self.client_session.request(
|
response = await self.get_client_session().request(
|
||||||
method=method,
|
method=method,
|
||||||
url=url,
|
url=url,
|
||||||
json=body,
|
json=body,
|
||||||
headers=headers,
|
headers=headers,
|
||||||
auth=aiohttp.BasicAuth("hub", current_app.config['HUB_TOKEN']),
|
|
||||||
verify_ssl=True,
|
verify_ssl=True,
|
||||||
)
|
)
|
||||||
except:
|
except:
|
||||||
@ -59,15 +69,92 @@ class MyHTTPClient:
|
|||||||
|
|
||||||
return HTTPResult(response.status, response_body)
|
return HTTPResult(response.status, response_body)
|
||||||
|
|
||||||
async def make_requests(self, online_hosts: List[OnlineHost], body: str) -> List(HTTPResult):
|
async def make_requests(self, online_hosts: List[OnlineHost], url_suffix: str, body: str, authorization_header=None) -> List[HTTPResult]:
|
||||||
tasks = []
|
tasks = []
|
||||||
# append to tasks in the same order as online_hosts
|
# append to tasks in the same order as online_hosts
|
||||||
for host in online_hosts:
|
for host in online_hosts:
|
||||||
tasks.append(
|
tasks.append(
|
||||||
self.post_json(url=host.url, body=body)
|
self.post_json(url=f"{host.url}/{url_suffix}", body=body, authorization_header=authorization_header)
|
||||||
)
|
)
|
||||||
# gather is like Promise.all from javascript, it returns a future which resolves to an array of results
|
# gather is like Promise.all from javascript, it returns a future which resolves to an array of results
|
||||||
# in the same order as the tasks that we passed in -- which were in the same order as online_hosts
|
# in the same order as the tasks that we passed in -- which were in the same order as online_hosts
|
||||||
results = await asyncio.gather(*tasks)
|
results = await asyncio.gather(*tasks)
|
||||||
|
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
# i lifted this direct from https://stackoverflow.com/a/58616001
|
||||||
|
# this is the bridge between Flask's one-thread-per-request world
|
||||||
|
# and aiohttp's event-loop based world
|
||||||
|
|
||||||
|
class EventLoopThread(threading.Thread):
|
||||||
|
loop = None
|
||||||
|
_count = itertools.count(0)
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
name = f"{type(self).__name__}-{next(self._count)}"
|
||||||
|
super().__init__(name=name, daemon=True)
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
loop, r, c, d = self.loop, False, True, False
|
||||||
|
if loop is not None:
|
||||||
|
r, c, d = loop.is_running(), loop.is_closed(), loop.get_debug()
|
||||||
|
return (
|
||||||
|
f"<{type(self).__name__} {self.name} id={self.ident} "
|
||||||
|
f"running={r} closed={c} debug={d}>"
|
||||||
|
)
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
self.loop = loop = asyncio.new_event_loop()
|
||||||
|
asyncio.set_event_loop(loop)
|
||||||
|
|
||||||
|
try:
|
||||||
|
loop.run_forever()
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
shutdown_asyncgens = loop.shutdown_asyncgens()
|
||||||
|
except AttributeError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
loop.run_until_complete(shutdown_asyncgens)
|
||||||
|
loop.close()
|
||||||
|
asyncio.set_event_loop(None)
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
loop, self.loop = self.loop, None
|
||||||
|
if loop is None:
|
||||||
|
return
|
||||||
|
loop.call_soon_threadsafe(loop.stop)
|
||||||
|
self.join()
|
||||||
|
|
||||||
|
_lock = threading.Lock()
|
||||||
|
_loop_thread = None
|
||||||
|
|
||||||
|
def get_event_loop():
|
||||||
|
global _loop_thread
|
||||||
|
|
||||||
|
if _loop_thread is None:
|
||||||
|
with _lock:
|
||||||
|
if _loop_thread is None:
|
||||||
|
_loop_thread = EventLoopThread()
|
||||||
|
_loop_thread.start()
|
||||||
|
# give the thread up to a second to produce a loop
|
||||||
|
deadline = time.time() + 1
|
||||||
|
while not _loop_thread.loop and time.time() < deadline:
|
||||||
|
time.sleep(0.001)
|
||||||
|
|
||||||
|
return _loop_thread.loop
|
||||||
|
|
||||||
|
def stop_event_loop():
|
||||||
|
global _loop_thread
|
||||||
|
with _lock:
|
||||||
|
if _loop_thread is not None:
|
||||||
|
_loop_thread.stop()
|
||||||
|
_loop_thread = None
|
||||||
|
|
||||||
|
def run_coroutine(coro):
|
||||||
|
"""Run the coroutine in the event loop running in a separate thread
|
||||||
|
Returns a Future, call Future.result() to get the output
|
||||||
|
"""
|
||||||
|
|
||||||
|
return asyncio.run_coroutine_threadsafe(coro, get_event_loop())
|
@ -4,23 +4,46 @@ from flask import current_app
|
|||||||
from flask import request
|
from flask import request
|
||||||
from werkzeug.exceptions import abort
|
from werkzeug.exceptions import abort
|
||||||
|
|
||||||
from capsulflask.db import get_model, my_exec_info_message
|
from capsulflask.db import get_model
|
||||||
|
from capsulflask.shared import my_exec_info_message, authorized_as_hub
|
||||||
|
|
||||||
bp = Blueprint("hub", __name__, url_prefix="/hub")
|
bp = Blueprint("hub", __name__, url_prefix="/hub")
|
||||||
|
|
||||||
def authorized_for_host(id):
|
def authorized_for_host(id):
|
||||||
|
if request.headers.get('Authorization'):
|
||||||
auth_header_value = request.headers.get('Authorization').replace("Bearer ", "")
|
auth_header_value = request.headers.get('Authorization').replace("Bearer ", "")
|
||||||
return get_model().authorized_for_host(id, auth_header_value)
|
return get_model().authorized_for_host(id, auth_header_value)
|
||||||
|
return False
|
||||||
|
|
||||||
@bp.route("/heartbeat/<string:host_id>", methods=("POST"))
|
@bp.route("/heartbeat-task", methods=("POST",))
|
||||||
|
def ping_all_hosts_task():
|
||||||
|
if authorized_as_hub(request.headers):
|
||||||
|
all_hosts = get_model().get_all_hosts()
|
||||||
|
current_app.logger.info(f"pinging {len(all_hosts)} hosts...")
|
||||||
|
authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}"
|
||||||
|
results = current_app.config["HTTP_CLIENT"].make_requests_sync(all_hosts, "/spoke/heartbeat", None, authorization_header=authorization_header)
|
||||||
|
for i in range(len(all_hosts)):
|
||||||
|
host = all_hosts[i]
|
||||||
|
result = results[i]
|
||||||
|
current_app.logger.info(f"response from {host.id} ({host.url}): {result.status_code} {result.body}")
|
||||||
|
if result.status_code == 200:
|
||||||
|
get_model().host_heartbeat(host.id)
|
||||||
|
|
||||||
|
return "ok"
|
||||||
|
else:
|
||||||
|
current_app.logger.info(f"/hub/heartbeat-task returned 401: invalid hub token")
|
||||||
|
return abort(401, "invalid hub token")
|
||||||
|
|
||||||
|
|
||||||
|
@bp.route("/heartbeat/<string:host_id>", methods=("POST",))
|
||||||
def heartbeat(host_id):
|
def heartbeat(host_id):
|
||||||
if authorized_for_host(host_id):
|
if authorized_for_host(host_id):
|
||||||
get_model().host_heartbeat(host_id)
|
return "ok"
|
||||||
else:
|
else:
|
||||||
current_app.logger.info(f"/hub/heartbeat/{host_id} returned 401: invalid token")
|
current_app.logger.info(f"/hub/heartbeat/{host_id} returned 401: invalid token")
|
||||||
return abort(401, "invalid host id or token")
|
return abort(401, "invalid host id or token")
|
||||||
|
|
||||||
@bp.route("/claim-operation/<int:operation_id>/<string:host_id>", methods=("POST"))
|
@bp.route("/claim-operation/<int:operation_id>/<string:host_id>", methods=("POST",))
|
||||||
def claim_operation(operation_id: int, host_id: str):
|
def claim_operation(operation_id: int, host_id: str):
|
||||||
if authorized_for_host(host_id):
|
if authorized_for_host(host_id):
|
||||||
exists = get_model().host_operation_exists(operation_id, host_id)
|
exists = get_model().host_operation_exists(operation_id, host_id)
|
||||||
|
@ -12,33 +12,9 @@ from time import sleep
|
|||||||
from os.path import join
|
from os.path import join
|
||||||
from subprocess import run
|
from subprocess import run
|
||||||
|
|
||||||
from capsulflask.db_model import OnlineHost
|
from capsulflask.db import get_model
|
||||||
from capsulflask.spoke_model import validate_capsul_id
|
|
||||||
from capsulflask.db import get_model, my_exec_info_message
|
|
||||||
from capsulflask.http_client import HTTPResult
|
from capsulflask.http_client import HTTPResult
|
||||||
|
from capsulflask.shared import VirtualizationInterface, VirtualMachine, OnlineHost, validate_capsul_id, my_exec_info_message
|
||||||
class VirtualMachine:
|
|
||||||
def __init__(self, id, host, ipv4=None, ipv6=None):
|
|
||||||
self.id = id
|
|
||||||
self.host = host
|
|
||||||
self.ipv4 = ipv4
|
|
||||||
self.ipv6 = ipv6
|
|
||||||
|
|
||||||
class VirtualizationInterface:
|
|
||||||
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def get(self, id: str) -> VirtualMachine:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def list_ids(self) -> list:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_public_keys: list):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def destroy(self, email: str, id: str):
|
|
||||||
pass
|
|
||||||
|
|
||||||
class MockHub(VirtualizationInterface):
|
class MockHub(VirtualizationInterface):
|
||||||
def capacity_avaliable(self, additional_ram_bytes):
|
def capacity_avaliable(self, additional_ram_bytes):
|
||||||
@ -65,7 +41,7 @@ class CapsulFlaskHub(VirtualizationInterface):
|
|||||||
def generic_operation(self, hosts: List[OnlineHost], payload: str, immediate_mode: bool) -> Tuple[int, List[HTTPResult]]:
|
def generic_operation(self, hosts: List[OnlineHost], payload: str, immediate_mode: bool) -> Tuple[int, List[HTTPResult]]:
|
||||||
operation_id = get_model().create_operation(hosts, payload)
|
operation_id = get_model().create_operation(hosts, payload)
|
||||||
authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}"
|
authorization_header = f"Bearer {current_app.config['HUB_TOKEN']}"
|
||||||
results = current_app.config["HTTP_CLIENT"].make_requests_sync(hosts, payload, authorization_header=authorization_header)
|
results = current_app.config["HTTP_CLIENT"].make_requests_sync(hosts, "/spoke/operation", payload, authorization_header=authorization_header)
|
||||||
for i in range(len(hosts)):
|
for i in range(len(hosts)):
|
||||||
host = hosts[i]
|
host = hosts[i]
|
||||||
result = results[i]
|
result = results[i]
|
||||||
@ -137,7 +113,7 @@ class CapsulFlaskHub(VirtualizationInterface):
|
|||||||
host = get_model().host_of_capsul(id)
|
host = get_model().host_of_capsul(id)
|
||||||
if host is not None:
|
if host is not None:
|
||||||
payload = json.dumps(dict(type="get", id=id))
|
payload = json.dumps(dict(type="get", id=id))
|
||||||
op = await self.generic_operation([host], payload, True)
|
op = self.generic_operation([host], payload, True)
|
||||||
results = op[1]
|
results = op[1]
|
||||||
for result in results:
|
for result in results:
|
||||||
try:
|
try:
|
||||||
@ -152,7 +128,7 @@ class CapsulFlaskHub(VirtualizationInterface):
|
|||||||
def list_ids(self) -> list:
|
def list_ids(self) -> list:
|
||||||
online_hosts = get_model().get_online_hosts()
|
online_hosts = get_model().get_online_hosts()
|
||||||
payload = json.dumps(dict(type="list_ids"))
|
payload = json.dumps(dict(type="list_ids"))
|
||||||
op = await self.generic_operation(online_hosts, payload, False)
|
op = self.generic_operation(online_hosts, payload, False)
|
||||||
operation_id = op[0]
|
operation_id = op[0]
|
||||||
results = op[1]
|
results = op[1]
|
||||||
to_return = []
|
to_return = []
|
||||||
@ -197,7 +173,7 @@ class CapsulFlaskHub(VirtualizationInterface):
|
|||||||
memory_mb=memory_mb,
|
memory_mb=memory_mb,
|
||||||
ssh_public_keys=ssh_public_keys,
|
ssh_public_keys=ssh_public_keys,
|
||||||
))
|
))
|
||||||
op = await self.generic_operation(online_hosts, payload, False)
|
op = self.generic_operation(online_hosts, payload, False)
|
||||||
operation_id = op[0]
|
operation_id = op[0]
|
||||||
results = op[1]
|
results = op[1]
|
||||||
number_of_assigned = 0
|
number_of_assigned = 0
|
||||||
@ -230,7 +206,7 @@ class CapsulFlaskHub(VirtualizationInterface):
|
|||||||
host = get_model().host_of_capsul(id)
|
host = get_model().host_of_capsul(id)
|
||||||
if host is not None:
|
if host is not None:
|
||||||
payload = json.dumps(dict(type="destroy", id=id))
|
payload = json.dumps(dict(type="destroy", id=id))
|
||||||
op = await self.generic_operation([host], payload, True)
|
op = self.generic_operation([host], payload, True)
|
||||||
results = op[1]
|
results = op[1]
|
||||||
result_json_string = "<no response from host>"
|
result_json_string = "<no response from host>"
|
||||||
for result in results:
|
for result in results:
|
||||||
|
@ -20,7 +20,8 @@ from werkzeug.exceptions import abort
|
|||||||
|
|
||||||
from capsulflask.auth import account_required
|
from capsulflask.auth import account_required
|
||||||
|
|
||||||
from capsulflask.db import get_model, my_exec_info_message
|
from capsulflask.db import get_model
|
||||||
|
from capsulflask.shared import my_exec_info_message
|
||||||
|
|
||||||
bp = Blueprint("payment", __name__, url_prefix="/payment")
|
bp = Blueprint("payment", __name__, url_prefix="/payment")
|
||||||
|
|
||||||
|
@ -7,7 +7,7 @@ CREATE TABLE hosts (
|
|||||||
token TEXT NOT NULL
|
token TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
INSERT INTO hosts (id, token) VALUES ('baikal', 'changeme');
|
INSERT INTO hosts (id, https_url, token) VALUES ('baikal', 'http://localhost:5000', 'changeme');
|
||||||
|
|
||||||
ALTER TABLE vms
|
ALTER TABLE vms
|
||||||
ADD COLUMN host TEXT REFERENCES hosts(id) ON DELETE RESTRICT DEFAULT 'baikal';
|
ADD COLUMN host TEXT REFERENCES hosts(id) ON DELETE RESTRICT DEFAULT 'baikal';
|
||||||
@ -16,7 +16,7 @@ CREATE TABLE operations (
|
|||||||
id SERIAL PRIMARY KEY ,
|
id SERIAL PRIMARY KEY ,
|
||||||
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
|
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
|
||||||
created TIMESTAMP NOT NULL DEFAULT NOW(),
|
created TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||||
payload TEXT NOT NULL,
|
payload TEXT NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE host_operation (
|
CREATE TABLE host_operation (
|
||||||
|
44
capsulflask/shared.py
Normal file
44
capsulflask/shared.py
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
import re
|
||||||
|
|
||||||
|
from flask import current_app
|
||||||
|
|
||||||
|
class OnlineHost:
|
||||||
|
def __init__(self, id: str, url: str):
|
||||||
|
self.id = id
|
||||||
|
self.url = url
|
||||||
|
|
||||||
|
class VirtualMachine:
|
||||||
|
def __init__(self, id, host, ipv4=None, ipv6=None):
|
||||||
|
self.id = id
|
||||||
|
self.host = host
|
||||||
|
self.ipv4 = ipv4
|
||||||
|
self.ipv6 = ipv6
|
||||||
|
|
||||||
|
class VirtualizationInterface:
|
||||||
|
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get(self, id: str) -> VirtualMachine:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def list_ids(self) -> list:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_public_keys: list):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def destroy(self, email: str, id: str):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def validate_capsul_id(id):
|
||||||
|
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
|
||||||
|
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
|
||||||
|
|
||||||
|
def authorized_as_hub(headers):
|
||||||
|
if headers.get('Authorization'):
|
||||||
|
auth_header_value = headers.get('Authorization').replace("Bearer ", "")
|
||||||
|
return auth_header_value == current_app.config["HUB_TOKEN"]
|
||||||
|
return False
|
||||||
|
|
||||||
|
def my_exec_info_message(exec_info):
|
||||||
|
return "{}: {}".format(".".join([exec_info[0].__module__, exec_info[0].__name__]), exec_info[1])
|
@ -7,17 +7,13 @@ from flask import request
|
|||||||
from flask.json import jsonify
|
from flask.json import jsonify
|
||||||
from werkzeug.exceptions import abort
|
from werkzeug.exceptions import abort
|
||||||
|
|
||||||
from capsulflask.db import my_exec_info_message
|
from capsulflask.shared import my_exec_info_message, authorized_as_hub
|
||||||
|
|
||||||
bp = Blueprint("spoke", __name__, url_prefix="/spoke")
|
bp = Blueprint("spoke", __name__, url_prefix="/spoke")
|
||||||
|
|
||||||
def authorized_as_hub(id):
|
@bp.route("/heartbeat", methods=("POST",))
|
||||||
auth_header_value = request.headers.get('Authorization').replace("Bearer ", "")
|
|
||||||
return auth_header_value == current_app.config["HUB_TOKEN"]
|
|
||||||
|
|
||||||
@bp.route("/heartbeat", methods=("POST"))
|
|
||||||
def heartbeat():
|
def heartbeat():
|
||||||
if authorized_as_hub(id):
|
if authorized_as_hub(request.headers):
|
||||||
url = f"{current_app.config['HUB_URL']}/hub/heartbeat/{current_app.config['SPOKE_HOST_ID']}"
|
url = f"{current_app.config['HUB_URL']}/hub/heartbeat/{current_app.config['SPOKE_HOST_ID']}"
|
||||||
authorization_header = f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}"
|
authorization_header = f"Bearer {current_app.config['SPOKE_HOST_TOKEN']}"
|
||||||
result = current_app.config['HTTP_CLIENT'].post_json_sync(url, body=None, authorization_header=authorization_header)
|
result = current_app.config['HTTP_CLIENT'].post_json_sync(url, body=None, authorization_header=authorization_header)
|
||||||
@ -36,9 +32,9 @@ def heartbeat():
|
|||||||
current_app.logger.info(f"/hosts/heartbeat returned 401: invalid hub token")
|
current_app.logger.info(f"/hosts/heartbeat returned 401: invalid hub token")
|
||||||
return abort(401, "invalid hub token")
|
return abort(401, "invalid hub token")
|
||||||
|
|
||||||
@bp.route("/operation", methods=("POST"))
|
@bp.route("/operation", methods=("POST",))
|
||||||
def operation():
|
def operation():
|
||||||
if authorized_as_hub(id):
|
if authorized_as_hub(request.headers):
|
||||||
request_body = request.json()
|
request_body = request.json()
|
||||||
handlers = {
|
handlers = {
|
||||||
"capacity_avaliable": handle_capacity_avaliable,
|
"capacity_avaliable": handle_capacity_avaliable,
|
||||||
|
@ -8,11 +8,8 @@ from subprocess import run
|
|||||||
|
|
||||||
from capsulflask.db import get_model
|
from capsulflask.db import get_model
|
||||||
|
|
||||||
from capsulflask.hub_model import VirtualizationInterface, VirtualMachine
|
from capsulflask.shared import VirtualizationInterface, VirtualMachine, validate_capsul_id
|
||||||
|
|
||||||
def validate_capsul_id(id):
|
|
||||||
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
|
|
||||||
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
|
|
||||||
|
|
||||||
class MockSpoke(VirtualizationInterface):
|
class MockSpoke(VirtualizationInterface):
|
||||||
def capacity_avaliable(self, additional_ram_bytes):
|
def capacity_avaliable(self, additional_ram_bytes):
|
||||||
|
Loading…
Reference in New Issue
Block a user