more managed ips work: cli sql improvements, added admin panel

This commit is contained in:
forest 2021-07-09 14:13:28 -05:00
parent e685c8a773
commit 862b14545b
10 changed files with 179 additions and 43 deletions

View File

@ -61,6 +61,7 @@ app.config.from_mapping(
MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""), MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"), MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"), ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"),
ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"),
PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"), PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
@ -143,6 +144,12 @@ try:
except: except:
app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info())) app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
# only start the scheduler and attempt to migrate the database if we are running the app.
# otherwise we are running a CLI command.
command_line = ' '.join(sys.argv)
is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
app.logger.info(f"is_running_server: {is_running_server}")
if app.config['HUB_MODE_ENABLED']: if app.config['HUB_MODE_ENABLED']:
@ -151,7 +158,7 @@ if app.config['HUB_MODE_ENABLED']:
# debug mode (flask reloader) runs two copies of the app. When running in debug mode, # debug mode (flask reloader) runs two copies of the app. When running in debug mode,
# we only want to start the scheduler one time. # we only want to start the scheduler one time.
if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
scheduler = BackgroundScheduler() scheduler = BackgroundScheduler()
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task" heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"} heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
@ -165,9 +172,9 @@ if app.config['HUB_MODE_ENABLED']:
app.config['HUB_MODEL'] = hub_model.MockHub() app.config['HUB_MODEL'] = hub_model.MockHub()
from capsulflask import db from capsulflask import db
db.init_app(app) db.init_app(app, is_running_server)
from capsulflask import auth, landing, console, payment, metrics, cli, hub_api from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
app.register_blueprint(landing.bp) app.register_blueprint(landing.bp)
app.register_blueprint(auth.bp) app.register_blueprint(auth.bp)
@ -176,6 +183,7 @@ if app.config['HUB_MODE_ENABLED']:
app.register_blueprint(metrics.bp) app.register_blueprint(metrics.bp)
app.register_blueprint(cli.bp) app.register_blueprint(cli.bp)
app.register_blueprint(hub_api.bp) app.register_blueprint(hub_api.bp)
app.register_blueprint(admin.bp)
app.add_url_rule("/", endpoint="index") app.add_url_rule("/", endpoint="index")

View File

@ -3,16 +3,9 @@ import sys
import json import json
import ipaddress import ipaddress
from datetime import datetime, timedelta from datetime import datetime, timedelta
from flask import Blueprint from flask import Blueprint, current_app, render_template, make_response
from flask import flash
from flask import current_app
from flask import g
from flask import request
from flask import session
from flask import render_template
from flask import redirect
from flask import url_for
from werkzeug.exceptions import abort from werkzeug.exceptions import abort
from nanoid import generate
from capsulflask.metrics import durations as metric_durations from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required from capsulflask.auth import admin_account_required
@ -25,18 +18,23 @@ bp = Blueprint("admin", __name__, url_prefix="/admin")
@admin_account_required @admin_account_required
def index(): def index():
hosts = get_model().list_hosts_with_networks() hosts = get_model().list_hosts_with_networks()
vms = get_model().all_non_deleted_vms() vms_by_host_and_network = get_model().all_non_deleted_vms_by_host_and_network()
operations = get_model().list_all_operations() network_display_width_px = float(250);
#operations = get_model().list_all_operations()
display_hosts = [] display_hosts = []
inline_styles = [f"""
.network-display {'{'}
width: {network_display_width_px}px;
{'}'}
"""]
for kv in hosts.items(): for kv in hosts.items():
name = kv[0] host_id = kv[0]
value = kv[1] value = kv[1]
display_host = dict(name=host_id, networks=value['networks'])
for network in value['networks']: for network in display_host['networks']:
network["network_name"]
ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False) ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False)
network_start_int = -1 network_start_int = -1
network_end_int = -1 network_end_int = -1
@ -49,9 +47,41 @@ def index():
network_end_int = int(ipv4_address) network_end_int = int(ipv4_address)
network['allocations'] = []
network_addresses_width = float((network_end_int-network_start_int)+1)
if host_id in vms_by_host_and_network:
if network['network_name'] in vms_by_host_and_network[host_id]:
for vm in vms_by_host_and_network[host_id][network['network_name']]:
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int < ip_address_int and ip_address_int < network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
inline_styles.append(
f"""
.{allocation} {'{'}
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
width: {network_display_width_px/network_addresses_width}px;
{'}'}
"""
)
network['allocations'].append(allocation)
else:
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
display_hosts.append(dict(name=name, last_health_check=value['last_health_check'])) display_hosts.append(display_host)
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
response_text = render_template(
"admin.html",
display_hosts=display_hosts,
network_display_width_px=network_display_width_px,
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles)
)
return render_template("admin.html", vms=mappedVms, has_vms=len(vms) > 0, created=created) response = make_response(response_text)
response.headers.set('Content-Type', 'text/html')
response.headers.set('Content-Security-Policy', f"default-src 'self'; style-src 'self' 'nonce-{csp_inline_style_nonce}'")
return response

View File

@ -40,7 +40,7 @@ def admin_account_required(view):
if session.get("account") is None or session.get("csrf-token") is None: if session.get("account") is None or session.get("csrf-token") is None:
return redirect(url_for("auth.login")) return redirect(url_for("auth.login"))
if session.get("account") not in current_app.config["ADMIN_EMAIL_ADDRESSES_CSV"].split(","): if session.get("account") not in current_app.config["ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES"].split(","):
return redirect(url_for("auth.login")) return redirect(url_for("auth.login"))
return view(**kwargs) return view(**kwargs)

View File

@ -23,9 +23,9 @@ from capsulflask import cli
bp = Blueprint("console", __name__, url_prefix="/console") bp = Blueprint("console", __name__, url_prefix="/console")
def makeCapsulId(): def make_capsul_id():
lettersAndNumbers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
return f"capsul-{lettersAndNumbers}" return f"capsul-{letters_n_nummers}"
def double_check_capsul_address(id, ipv4, get_ssh_host_keys): def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
try: try:
@ -244,7 +244,7 @@ def create():
""") """)
if len(errors) == 0: if len(errors) == 0:
id = makeCapsulId() id = make_capsul_id()
get_model().create_vm( get_model().create_vm(
email=session["account"], email=session["account"],
id=id, id=id,

View File

@ -10,7 +10,7 @@ from flask import g
from capsulflask.db_model import DBModel from capsulflask.db_model import DBModel
from capsulflask.shared import my_exec_info_message from capsulflask.shared import my_exec_info_message
def init_app(app): def init_app(app, is_running_server):
app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool( app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool(
1, 1,
@ -18,6 +18,14 @@ def init_app(app):
app.config['POSTGRES_CONNECTION_PARAMETERS'] app.config['POSTGRES_CONNECTION_PARAMETERS']
) )
# tell the app to clean up the DB connection when shutting down.
app.teardown_appcontext(close_db)
# only run the migrations if we are running the server.
# If we are just running a cli command (e.g. to fix a broken migration 😅), skip it
if not is_running_server:
return
schemaMigrations = {} schemaMigrations = {}
schemaMigrationsPath = join(app.root_path, 'schema_migrations') schemaMigrationsPath = join(app.root_path, 'schema_migrations')
app.logger.info("loading schema migration scripts from {}".format(schemaMigrationsPath)) app.logger.info("loading schema migration scripts from {}".format(schemaMigrationsPath))
@ -35,7 +43,7 @@ def init_app(app):
hasSchemaVersionTable = False hasSchemaVersionTable = False
actionWasTaken = False actionWasTaken = False
schemaVersion = 0 schemaVersion = 0
desiredSchemaVersion = 15 desiredSchemaVersion = 16
cursor = connection.cursor() cursor = connection.cursor()
@ -103,7 +111,7 @@ def init_app(app):
("schema migration completed." if actionWasTaken else "schema is already up to date. "), schemaVersion ("schema migration completed." if actionWasTaken else "schema is already up to date. "), schemaVersion
)) ))
app.teardown_appcontext(close_db)
def get_model(): def get_model():

View File

@ -56,9 +56,23 @@ class DBModel:
# ------ VM & ACCOUNT MANAGEMENT --------- # ------ VM & ACCOUNT MANAGEMENT ---------
def all_non_deleted_vms(self): def all_non_deleted_vms_by_host_and_network(self):
self.cursor.execute("SELECT id, host, network_name, last_seen_ipv4, last_seen_ipv6 FROM vms WHERE deleted IS NULL") self.cursor.execute("SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL")
return list(map(lambda x: dict(id=x[0], host=x[1], network_name=x[2], last_seen_ipv4=x[3], last_seen_ipv6=x[4]), self.cursor.fetchall()))
hosts = dict()
for row in self.cursor.fetchall():
host_id = row[1]
network_name = row[2]
if host_id not in hosts:
hosts[host_id] = dict()
if network_name not in hosts[host_id]:
hosts[host_id][network_name] = []
hosts[host_id][network_name].append(
dict(id=row[0], public_ipv4=row[3], public_ipv6=row[4])
)
return hosts
def all_non_deleted_vm_ids(self): def all_non_deleted_vm_ids(self):
self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL") self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
@ -108,7 +122,7 @@ class DBModel:
def list_vms_for_account(self, email): def list_vms_for_account(self, email):
self.cursor.execute(""" self.cursor.execute("""
SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
FROM vms JOIN vm_sizes on vms.size = vm_sizes.id FROM vms JOIN vm_sizes on vms.size = vm_sizes.id
WHERE vms.email = %s""", WHERE vms.email = %s""",
(email, ) (email, )
@ -119,7 +133,7 @@ class DBModel:
)) ))
def update_vm_ip(self, email, id, ipv4): def update_vm_ip(self, email, id, ipv4):
self.cursor.execute("UPDATE vms SET last_seen_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id)) self.cursor.execute("UPDATE vms SET public_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
self.connection.commit() self.connection.commit()
def update_vm_ssh_host_keys(self, email, id, ssh_host_keys): def update_vm_ssh_host_keys(self, email, id, ssh_host_keys):
@ -155,7 +169,7 @@ class DBModel:
def get_vm_detail(self, email, id): def get_vm_detail(self, email, id):
self.cursor.execute(""" self.cursor.execute("""
SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, os_images.description, vms.created, vms.deleted, SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted,
vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
FROM vms FROM vms
JOIN os_images on vms.os = os_images.id JOIN os_images on vms.os = os_images.id

View File

@ -1,6 +1,6 @@
DROP TABLE host_network; DROP TABLE host_network;
ALTER TABLE vms DROP COLUMN network; ALTER TABLE vms DROP COLUMN network_name;
UPDATE schemaversion SET version = 15; UPDATE schemaversion SET version = 15;

View File

@ -1,14 +1,25 @@
CREATE TABLE host_network ( CREATE TABLE host_network (
public_ipv4_cidr_block TEXT PRIMARY KEY NOT NULL, public_ipv4_cidr_block TEXT NOT NULL,
network_name TEXT NOT NULL, network_name TEXT NOT NULL,
host TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT, host TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT,
PRIMARY KEY (host, network_name)
); );
INSERT INTO host_network (public_ipv4_cidr_block, network_name, host) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
('baikal', 'virbr2', '69.61.2.194/26');
ALTER TABLE vms ADD COLUMN network_name TEXT NOT NULL; INSERT INTO host_network (host, network_name, public_ipv4_cidr_block) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
('baikal', 'virbr2', '69.61.2.194/26');
ALTER TABLE vms RENAME COLUMN last_seen_ipv4 TO public_ipv4;
ALTER TABLE vms RENAME COLUMN last_seen_ipv6 TO public_ipv6;
ALTER TABLE vms ADD COLUMN network_name TEXT;
UPDATE vms SET network_name = 'virbr1' WHERE public_ipv6 < '69.61.2.192';
UPDATE vms SET network_name = 'virbr2' WHERE public_ipv6 >= '69.61.2.192';
ALTER TABLE vms ALTER COLUMN network_name SET NOT NULL;
ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT; ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
UPDATE schemaversion SET version = 16; UPDATE schemaversion SET version = 16;

View File

@ -356,3 +356,26 @@ footer {
font-size: 0.8em; font-size: 0.8em;
} }
.network-row {
background-color: #777e7350;
padding: 5px;
margin-top: 5px;
}
.network-display {
height: 1em;
border: 1px solid #777e73;
}
.network-display div {
position: absolute;
}
.network-display div div {
top: 1px;
height: calc(1em - 2px);
background-color: rgba(221, 169, 56, 0.8);
border: 1px solid rgba(255, 223, 155, 0.8);
box-sizing: border-box;
position: relative;
}

View File

@ -0,0 +1,42 @@
{% extends 'base.html' %}
{% block title %}Capsul Admin{% endblock %}
{% block content %}
<style nonce="{{csp_inline_style_nonce}}">
{{inline_style}}
</style>
<div class="row third-margin">
<h1>Capsul Admin</h1>
</div>
<div class="third-margin">
{% for display_host in display_hosts %}
<div class="row">
<h1>{{ display_host["name"] }}</h1>
</div>
{% for network in display_host["networks"] %}
<div class="row network-row">
<i>{{ network["network_name"] }}</i>
<span>{{ network["public_ipv4_cidr_block"] }}</span>
<div class="network-display">
{% for allocation in network["allocations"] %}
{# This outer div is used as an abs position container & selected by CSS so don't remove it pls. #}
<div>
<div class="{{allocation}}">
</div>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
<hr/>
{% endfor %}
</div>
{% endblock %}
{% block pagesource %}/templates/admin.html{% endblock %}