autonomic-cooperative/capsul-flask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

more managed ips work: cli sql improvements, added admin panel

Browse Source
This commit is contained in:
forest
2021-07-09 14:13:28 -05:00
parent e685c8a773
commit 862b14545b
10 changed files with 179 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
capsulflask/__init__.py
View File

@ -61,6 +61,7 @@ app.config.from_mapping(
MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"),
ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"),
PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
@ -143,6 +144,12 @@ try:
except:
app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
# only start the scheduler and attempt to migrate the database if we are running the app.
# otherwise we are running a CLI command.
command_line = ' '.join(sys.argv)
is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
app.logger.info(f"is_running_server: {is_running_server}")
if app.config['HUB_MODE_ENABLED']:
@ -151,7 +158,7 @@ if app.config['HUB_MODE_ENABLED']:
# debug mode (flask reloader) runs two copies of the app. When running in debug mode,
# we only want to start the scheduler one time.
if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
scheduler = BackgroundScheduler()
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
@ -163,11 +170,11 @@ if app.config['HUB_MODE_ENABLED']:
else:
app.config['HUB_MODEL'] = hub_model.MockHub()
from capsulflask import db
db.init_app(app)
from capsulflask import auth, landing, console, payment, metrics, cli, hub_api
from capsulflask import db
db.init_app(app, is_running_server)
from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
app.register_blueprint(landing.bp)
app.register_blueprint(auth.bp)
@ -176,6 +183,7 @@ if app.config['HUB_MODE_ENABLED']:
app.register_blueprint(metrics.bp)
app.register_blueprint(cli.bp)
app.register_blueprint(hub_api.bp)
app.register_blueprint(admin.bp)
app.add_url_rule("/", endpoint="index")

68
capsulflask/admin.py
View File

@ -3,16 +3,9 @@ import sys
import json
import ipaddress
from datetime import datetime, timedelta
from flask import Blueprint
from flask import flash
from flask import current_app
from flask import g
from flask import request
from flask import session
from flask import render_template
from flask import redirect
from flask import url_for
from flask import Blueprint, current_app, render_template, make_response
from werkzeug.exceptions import abort
from nanoid import generate
from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required
@ -25,18 +18,23 @@ bp = Blueprint("admin", __name__, url_prefix="/admin")
@admin_account_required
def index():
hosts = get_model().list_hosts_with_networks()
vms = get_model().all_non_deleted_vms()
operations = get_model().list_all_operations()
vms_by_host_and_network = get_model().all_non_deleted_vms_by_host_and_network()
network_display_width_px = float(250);
#operations = get_model().list_all_operations()
display_hosts = []
inline_styles = [f"""
.network-display {'{'}
width: {network_display_width_px}px;
{'}'}
"""]
for kv in hosts.items():
name = kv[0]
host_id = kv[0]
value = kv[1]
for network in value['networks']:
network["network_name"]
display_host = dict(name=host_id, networks=value['networks'])
for network in display_host['networks']:
ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False)
network_start_int = -1
network_end_int = -1
@ -48,10 +46,42 @@ def index():
network_start_int = int(ipv4_address)
network_end_int = int(ipv4_address)
network['allocations'] = []
network_addresses_width = float((network_end_int-network_start_int)+1)
display_hosts.append(dict(name=name, last_health_check=value['last_health_check']))
if host_id in vms_by_host_and_network:
if network['network_name'] in vms_by_host_and_network[host_id]:
for vm in vms_by_host_and_network[host_id][network['network_name']]:
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int < ip_address_int and ip_address_int < network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
inline_styles.append(
f"""
.{allocation} {'{'}
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
width: {network_display_width_px/network_addresses_width}px;
{'}'}
"""
)
network['allocations'].append(allocation)
else:
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
return render_template("admin.html", vms=mappedVms, has_vms=len(vms) > 0, created=created)
display_hosts.append(display_host)
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
response_text = render_template(
"admin.html",
display_hosts=display_hosts,
network_display_width_px=network_display_width_px,
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles)
)
response = make_response(response_text)
response.headers.set('Content-Type', 'text/html')
response.headers.set('Content-Security-Policy', f"default-src 'self'; style-src 'self' 'nonce-{csp_inline_style_nonce}'")
return response

2
capsulflask/auth.py
View File

@ -40,7 +40,7 @@ def admin_account_required(view):
if session.get("account") is None or session.get("csrf-token") is None:
return redirect(url_for("auth.login"))
if session.get("account") not in current_app.config["ADMIN_EMAIL_ADDRESSES_CSV"].split(","):
if session.get("account") not in current_app.config["ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES"].split(","):
return redirect(url_for("auth.login"))
return view(**kwargs)

8
capsulflask/console.py
View File

@ -23,9 +23,9 @@ from capsulflask import cli
bp = Blueprint("console", __name__, url_prefix="/console")
def makeCapsulId():
lettersAndNumbers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
return f"capsul-{lettersAndNumbers}"
def make_capsul_id():
letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
return f"capsul-{letters_n_nummers}"
def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
try:
@ -244,7 +244,7 @@ def create():
""")
if len(errors) == 0:
id = makeCapsulId()
id = make_capsul_id()
get_model().create_vm(
email=session["account"],
id=id,

14
capsulflask/db.py
View File

@ -10,7 +10,7 @@ from flask import g
from capsulflask.db_model import DBModel
from capsulflask.shared import my_exec_info_message
def init_app(app):
def init_app(app, is_running_server):
app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool(
1,
@ -18,6 +18,14 @@ def init_app(app):
app.config['POSTGRES_CONNECTION_PARAMETERS']
)
# tell the app to clean up the DB connection when shutting down.
app.teardown_appcontext(close_db)
# only run the migrations if we are running the server.
# If we are just running a cli command (e.g. to fix a broken migration 😅), skip it
if not is_running_server:
return
schemaMigrations = {}
schemaMigrationsPath = join(app.root_path, 'schema_migrations')
app.logger.info("loading schema migration scripts from {}".format(schemaMigrationsPath))
@ -35,7 +43,7 @@ def init_app(app):
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 15
desiredSchemaVersion = 16
cursor = connection.cursor()
@ -103,7 +111,7 @@ def init_app(app):
("schema migration completed." if actionWasTaken else "schema is already up to date. "), schemaVersion
))
app.teardown_appcontext(close_db)
def get_model():

26
capsulflask/db_model.py
View File

@ -56,9 +56,23 @@ class DBModel:
# ------ VM & ACCOUNT MANAGEMENT ---------
def all_non_deleted_vms(self):
self.cursor.execute("SELECT id, host, network_name, last_seen_ipv4, last_seen_ipv6 FROM vms WHERE deleted IS NULL")
return list(map(lambda x: dict(id=x[0], host=x[1], network_name=x[2], last_seen_ipv4=x[3], last_seen_ipv6=x[4]), self.cursor.fetchall()))
def all_non_deleted_vms_by_host_and_network(self):
self.cursor.execute("SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL")
hosts = dict()
for row in self.cursor.fetchall():
host_id = row[1]
network_name = row[2]
if host_id not in hosts:
hosts[host_id] = dict()
if network_name not in hosts[host_id]:
hosts[host_id][network_name] = []
hosts[host_id][network_name].append(
dict(id=row[0], public_ipv4=row[3], public_ipv6=row[4])
)
return hosts
def all_non_deleted_vm_ids(self):
self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
@ -108,7 +122,7 @@ class DBModel:
def list_vms_for_account(self, email):
self.cursor.execute("""
SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
FROM vms JOIN vm_sizes on vms.size = vm_sizes.id
WHERE vms.email = %s""",
(email, )
@ -119,7 +133,7 @@ class DBModel:
))
def update_vm_ip(self, email, id, ipv4):
self.cursor.execute("UPDATE vms SET last_seen_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
self.cursor.execute("UPDATE vms SET public_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
self.connection.commit()
def update_vm_ssh_host_keys(self, email, id, ssh_host_keys):
@ -155,7 +169,7 @@ class DBModel:
def get_vm_detail(self, email, id):
self.cursor.execute("""
SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, os_images.description, vms.created, vms.deleted,
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted,
vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
FROM vms
JOIN os_images on vms.os = os_images.id

2
capsulflask/schema_migrations/16_down_managed_ips.sql
View File

@ -1,6 +1,6 @@
DROP TABLE host_network;
ALTER TABLE vms DROP COLUMN network;
ALTER TABLE vms DROP COLUMN network_name;
UPDATE schemaversion SET version = 15;

19
capsulflask/schema_migrations/16_up_managed_ips.sql
View File

@ -1,14 +1,25 @@
CREATE TABLE host_network (
public_ipv4_cidr_block TEXT PRIMARY KEY NOT NULL,
public_ipv4_cidr_block TEXT NOT NULL,
network_name TEXT NOT NULL,
host TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT,
PRIMARY KEY (host, network_name)
);
INSERT INTO host_network (public_ipv4_cidr_block, network_name, host) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
('baikal', 'virbr2', '69.61.2.194/26');
ALTER TABLE vms ADD COLUMN network_name TEXT NOT NULL;
INSERT INTO host_network (host, network_name, public_ipv4_cidr_block) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
('baikal', 'virbr2', '69.61.2.194/26');
ALTER TABLE vms RENAME COLUMN last_seen_ipv4 TO public_ipv4;
ALTER TABLE vms RENAME COLUMN last_seen_ipv6 TO public_ipv6;
ALTER TABLE vms ADD COLUMN network_name TEXT;
UPDATE vms SET network_name = 'virbr1' WHERE public_ipv6 < '69.61.2.192';
UPDATE vms SET network_name = 'virbr2' WHERE public_ipv6 >= '69.61.2.192';
ALTER TABLE vms ALTER COLUMN network_name SET NOT NULL;
ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
UPDATE schemaversion SET version = 16;

23
capsulflask/static/style.css
View File

@ -356,3 +356,26 @@ footer {
font-size: 0.8em;
}
.network-row {
background-color: #777e7350;
padding: 5px;
margin-top: 5px;
}
.network-display {
height: 1em;
border: 1px solid #777e73;
}
.network-display div {
position: absolute;
}
.network-display div div {
top: 1px;
height: calc(1em - 2px);
background-color: rgba(221, 169, 56, 0.8);
border: 1px solid rgba(255, 223, 155, 0.8);
box-sizing: border-box;
position: relative;
}

42
capsulflask/templates/admin.html Normal file
View File

@ -0,0 +1,42 @@
{% extends 'base.html' %}
{% block title %}Capsul Admin{% endblock %}
{% block content %}
<style nonce="{{csp_inline_style_nonce}}">
{{inline_style}}
</style>
<div class="row third-margin">
<h1>Capsul Admin</h1>
</div>
<div class="third-margin">
{% for display_host in display_hosts %}
<div class="row">
<h1>{{ display_host["name"] }}</h1>
</div>
{% for network in display_host["networks"] %}
<div class="row network-row">
<i>{{ network["network_name"] }}</i>
<span>{{ network["public_ipv4_cidr_block"] }}</span>
<div class="network-display">
{% for allocation in network["allocations"] %}
{# This outer div is used as an abs position container & selected by CSS so don't remove it pls. #}
<div>
<div class="{{allocation}}">
</div>
</div>
{% endfor %}
</div>
</div>
{% endfor %}
<hr/>
{% endfor %}
</div>
{% endblock %}
{% block pagesource %}/templates/admin.html{% endblock %}