From 862b14545baa281e7785cb5bc8aff26da5e396a4 Mon Sep 17 00:00:00 2001 From: forest Date: Fri, 9 Jul 2021 14:13:28 -0500 Subject: [PATCH] more managed ips work: cli sql improvements, added admin panel --- capsulflask/__init__.py | 18 +++-- capsulflask/admin.py | 68 +++++++++++++------ capsulflask/auth.py | 2 +- capsulflask/console.py | 8 +-- capsulflask/db.py | 14 +++- capsulflask/db_model.py | 26 +++++-- .../schema_migrations/16_down_managed_ips.sql | 2 +- .../schema_migrations/16_up_managed_ips.sql | 19 ++++-- capsulflask/static/style.css | 23 +++++++ capsulflask/templates/admin.html | 42 ++++++++++++ 10 files changed, 179 insertions(+), 43 deletions(-) create mode 100644 capsulflask/templates/admin.html diff --git a/capsulflask/__init__.py b/capsulflask/__init__.py index 1a2a6ff..1c4e4d5 100644 --- a/capsulflask/__init__.py +++ b/capsulflask/__init__.py @@ -61,6 +61,7 @@ app.config.from_mapping( MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""), MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"), ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"), + ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"), PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"), @@ -143,6 +144,12 @@ try: except: app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info())) +# only start the scheduler and attempt to migrate the database if we are running the app. +# otherwise we are running a CLI command. +command_line = ' '.join(sys.argv) +is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line) + +app.logger.info(f"is_running_server: {is_running_server}") if app.config['HUB_MODE_ENABLED']: @@ -151,7 +158,7 @@ if app.config['HUB_MODE_ENABLED']: # debug mode (flask reloader) runs two copies of the app. When running in debug mode, # we only want to start the scheduler one time. - if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': + if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'): scheduler = BackgroundScheduler() heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task" heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"} @@ -163,11 +170,11 @@ if app.config['HUB_MODE_ENABLED']: else: app.config['HUB_MODEL'] = hub_model.MockHub() - - from capsulflask import db - db.init_app(app) - from capsulflask import auth, landing, console, payment, metrics, cli, hub_api + from capsulflask import db + db.init_app(app, is_running_server) + + from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin app.register_blueprint(landing.bp) app.register_blueprint(auth.bp) @@ -176,6 +183,7 @@ if app.config['HUB_MODE_ENABLED']: app.register_blueprint(metrics.bp) app.register_blueprint(cli.bp) app.register_blueprint(hub_api.bp) + app.register_blueprint(admin.bp) app.add_url_rule("/", endpoint="index") diff --git a/capsulflask/admin.py b/capsulflask/admin.py index 0a9cba4..e34b547 100644 --- a/capsulflask/admin.py +++ b/capsulflask/admin.py @@ -3,16 +3,9 @@ import sys import json import ipaddress from datetime import datetime, timedelta -from flask import Blueprint -from flask import flash -from flask import current_app -from flask import g -from flask import request -from flask import session -from flask import render_template -from flask import redirect -from flask import url_for +from flask import Blueprint, current_app, render_template, make_response from werkzeug.exceptions import abort +from nanoid import generate from capsulflask.metrics import durations as metric_durations from capsulflask.auth import admin_account_required @@ -25,18 +18,23 @@ bp = Blueprint("admin", __name__, url_prefix="/admin") @admin_account_required def index(): hosts = get_model().list_hosts_with_networks() - vms = get_model().all_non_deleted_vms() - operations = get_model().list_all_operations() + vms_by_host_and_network = get_model().all_non_deleted_vms_by_host_and_network() + network_display_width_px = float(250); + #operations = get_model().list_all_operations() display_hosts = [] + inline_styles = [f""" + .network-display {'{'} + width: {network_display_width_px}px; + {'}'} + """] for kv in hosts.items(): - name = kv[0] + host_id = kv[0] value = kv[1] - - for network in value['networks']: - network["network_name"] + display_host = dict(name=host_id, networks=value['networks']) + for network in display_host['networks']: ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False) network_start_int = -1 network_end_int = -1 @@ -48,10 +46,42 @@ def index(): network_start_int = int(ipv4_address) network_end_int = int(ipv4_address) - + network['allocations'] = [] + network_addresses_width = float((network_end_int-network_start_int)+1) - display_hosts.append(dict(name=name, last_health_check=value['last_health_check'])) - + if host_id in vms_by_host_and_network: + if network['network_name'] in vms_by_host_and_network[host_id]: + for vm in vms_by_host_and_network[host_id][network['network_name']]: + ip_address_int = int(ipaddress.ip_address(vm['public_ipv4'])) + if network_start_int < ip_address_int and ip_address_int < network_end_int: + allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}" + inline_styles.append( + f""" + .{allocation} {'{'} + left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px; + width: {network_display_width_px/network_addresses_width}px; + {'}'} + """ + ) + network['allocations'].append(allocation) + else: + current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}") - return render_template("admin.html", vms=mappedVms, has_vms=len(vms) > 0, created=created) + display_hosts.append(display_host) + + csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) + response_text = render_template( + "admin.html", + display_hosts=display_hosts, + network_display_width_px=network_display_width_px, + csp_inline_style_nonce=csp_inline_style_nonce, + inline_style='\n'.join(inline_styles) + ) + + response = make_response(response_text) + + response.headers.set('Content-Type', 'text/html') + response.headers.set('Content-Security-Policy', f"default-src 'self'; style-src 'self' 'nonce-{csp_inline_style_nonce}'") + + return response diff --git a/capsulflask/auth.py b/capsulflask/auth.py index 5244a24..7bcdb1a 100644 --- a/capsulflask/auth.py +++ b/capsulflask/auth.py @@ -40,7 +40,7 @@ def admin_account_required(view): if session.get("account") is None or session.get("csrf-token") is None: return redirect(url_for("auth.login")) - if session.get("account") not in current_app.config["ADMIN_EMAIL_ADDRESSES_CSV"].split(","): + if session.get("account") not in current_app.config["ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES"].split(","): return redirect(url_for("auth.login")) return view(**kwargs) diff --git a/capsulflask/console.py b/capsulflask/console.py index 409bd7a..55e0ffa 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -23,9 +23,9 @@ from capsulflask import cli bp = Blueprint("console", __name__, url_prefix="/console") -def makeCapsulId(): - lettersAndNumbers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) - return f"capsul-{lettersAndNumbers}" +def make_capsul_id(): + letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) + return f"capsul-{letters_n_nummers}" def double_check_capsul_address(id, ipv4, get_ssh_host_keys): try: @@ -244,7 +244,7 @@ def create(): """) if len(errors) == 0: - id = makeCapsulId() + id = make_capsul_id() get_model().create_vm( email=session["account"], id=id, diff --git a/capsulflask/db.py b/capsulflask/db.py index 197a221..57137b1 100644 --- a/capsulflask/db.py +++ b/capsulflask/db.py @@ -10,7 +10,7 @@ from flask import g from capsulflask.db_model import DBModel from capsulflask.shared import my_exec_info_message -def init_app(app): +def init_app(app, is_running_server): app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool( 1, @@ -18,6 +18,14 @@ def init_app(app): app.config['POSTGRES_CONNECTION_PARAMETERS'] ) + # tell the app to clean up the DB connection when shutting down. + app.teardown_appcontext(close_db) + + # only run the migrations if we are running the server. + # If we are just running a cli command (e.g. to fix a broken migration 😅), skip it + if not is_running_server: + return + schemaMigrations = {} schemaMigrationsPath = join(app.root_path, 'schema_migrations') app.logger.info("loading schema migration scripts from {}".format(schemaMigrationsPath)) @@ -35,7 +43,7 @@ def init_app(app): hasSchemaVersionTable = False actionWasTaken = False schemaVersion = 0 - desiredSchemaVersion = 15 + desiredSchemaVersion = 16 cursor = connection.cursor() @@ -103,7 +111,7 @@ def init_app(app): ("schema migration completed." if actionWasTaken else "schema is already up to date. "), schemaVersion )) - app.teardown_appcontext(close_db) + def get_model(): diff --git a/capsulflask/db_model.py b/capsulflask/db_model.py index 6f134d8..fcd99f4 100644 --- a/capsulflask/db_model.py +++ b/capsulflask/db_model.py @@ -56,9 +56,23 @@ class DBModel: # ------ VM & ACCOUNT MANAGEMENT --------- - def all_non_deleted_vms(self): - self.cursor.execute("SELECT id, host, network_name, last_seen_ipv4, last_seen_ipv6 FROM vms WHERE deleted IS NULL") - return list(map(lambda x: dict(id=x[0], host=x[1], network_name=x[2], last_seen_ipv4=x[3], last_seen_ipv6=x[4]), self.cursor.fetchall())) + def all_non_deleted_vms_by_host_and_network(self): + self.cursor.execute("SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL") + + hosts = dict() + for row in self.cursor.fetchall(): + host_id = row[1] + network_name = row[2] + if host_id not in hosts: + hosts[host_id] = dict() + if network_name not in hosts[host_id]: + hosts[host_id][network_name] = [] + + hosts[host_id][network_name].append( + dict(id=row[0], public_ipv4=row[3], public_ipv6=row[4]) + ) + + return hosts def all_non_deleted_vm_ids(self): self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL") @@ -108,7 +122,7 @@ class DBModel: def list_vms_for_account(self, email): self.cursor.execute(""" - SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month + SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month FROM vms JOIN vm_sizes on vms.size = vm_sizes.id WHERE vms.email = %s""", (email, ) @@ -119,7 +133,7 @@ class DBModel: )) def update_vm_ip(self, email, id, ipv4): - self.cursor.execute("UPDATE vms SET last_seen_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id)) + self.cursor.execute("UPDATE vms SET public_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id)) self.connection.commit() def update_vm_ssh_host_keys(self, email, id, ssh_host_keys): @@ -155,7 +169,7 @@ class DBModel: def get_vm_detail(self, email, id): self.cursor.execute(""" - SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, os_images.description, vms.created, vms.deleted, + SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted, vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month FROM vms JOIN os_images on vms.os = os_images.id diff --git a/capsulflask/schema_migrations/16_down_managed_ips.sql b/capsulflask/schema_migrations/16_down_managed_ips.sql index fb0f031..b6a4d0f 100644 --- a/capsulflask/schema_migrations/16_down_managed_ips.sql +++ b/capsulflask/schema_migrations/16_down_managed_ips.sql @@ -1,6 +1,6 @@ DROP TABLE host_network; -ALTER TABLE vms DROP COLUMN network; +ALTER TABLE vms DROP COLUMN network_name; UPDATE schemaversion SET version = 15; diff --git a/capsulflask/schema_migrations/16_up_managed_ips.sql b/capsulflask/schema_migrations/16_up_managed_ips.sql index 695c46c..968ce16 100644 --- a/capsulflask/schema_migrations/16_up_managed_ips.sql +++ b/capsulflask/schema_migrations/16_up_managed_ips.sql @@ -1,14 +1,25 @@ CREATE TABLE host_network ( - public_ipv4_cidr_block TEXT PRIMARY KEY NOT NULL, + public_ipv4_cidr_block TEXT NOT NULL, network_name TEXT NOT NULL, host TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT, + PRIMARY KEY (host, network_name) ); -INSERT INTO host_network (public_ipv4_cidr_block, network_name, host) VALUES ('baikal', 'virbr1', '69.61.2.162/27'), - ('baikal', 'virbr2', '69.61.2.194/26'); -ALTER TABLE vms ADD COLUMN network_name TEXT NOT NULL; +INSERT INTO host_network (host, network_name, public_ipv4_cidr_block) VALUES ('baikal', 'virbr1', '69.61.2.162/27'), + ('baikal', 'virbr2', '69.61.2.194/26'); +ALTER TABLE vms RENAME COLUMN last_seen_ipv4 TO public_ipv4; +ALTER TABLE vms RENAME COLUMN last_seen_ipv6 TO public_ipv6; +ALTER TABLE vms ADD COLUMN network_name TEXT; + +UPDATE vms SET network_name = 'virbr1' WHERE public_ipv6 < '69.61.2.192'; +UPDATE vms SET network_name = 'virbr2' WHERE public_ipv6 >= '69.61.2.192'; + +ALTER TABLE vms ALTER COLUMN network_name SET NOT NULL; + ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT; + UPDATE schemaversion SET version = 16; + diff --git a/capsulflask/static/style.css b/capsulflask/static/style.css index 34321e6..f4a2d32 100644 --- a/capsulflask/static/style.css +++ b/capsulflask/static/style.css @@ -356,3 +356,26 @@ footer { font-size: 0.8em; } +.network-row { + background-color: #777e7350; + padding: 5px; + margin-top: 5px; +} + +.network-display { + height: 1em; + border: 1px solid #777e73; +} + +.network-display div { + position: absolute; +} + +.network-display div div { + top: 1px; + height: calc(1em - 2px); + background-color: rgba(221, 169, 56, 0.8); + border: 1px solid rgba(255, 223, 155, 0.8); + box-sizing: border-box; + position: relative; +} \ No newline at end of file diff --git a/capsulflask/templates/admin.html b/capsulflask/templates/admin.html new file mode 100644 index 0000000..a52b02b --- /dev/null +++ b/capsulflask/templates/admin.html @@ -0,0 +1,42 @@ +{% extends 'base.html' %} + +{% block title %}Capsul Admin{% endblock %} + +{% block content %} + +
+

Capsul Admin

+
+
+ {% for display_host in display_hosts %} +
+

{{ display_host["name"] }}

+
+ {% for network in display_host["networks"] %} +
+ {{ network["network_name"] }} + {{ network["public_ipv4_cidr_block"] }} +
+ {% for allocation in network["allocations"] %} + + {# This outer div is used as an abs position container & selected by CSS so don't remove it pls. #} +
+ +
+
+ +
+ + {% endfor %} +
+
+ {% endfor %} + +
+ {% endfor %} +
+{% endblock %} + +{% block pagesource %}/templates/admin.html{% endblock %}