more managed ips work: cli sql improvements, added admin panel

capsulflask/__init__.py

@@ -61,6 +61,7 @@ app.config.from_mapping(
   MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
   MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
   ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"),
+  ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"),
 
   PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
 
@@ -143,6 +144,12 @@ try:
 except:
   app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
 
+# only start the scheduler and attempt to migrate the database if we are running the app.
+# otherwise we are running a CLI command.
+command_line = ' '.join(sys.argv)
+is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
+
+app.logger.info(f"is_running_server: {is_running_server}")
 
 if app.config['HUB_MODE_ENABLED']:
 
@@ -151,7 +158,7 @@ if app.config['HUB_MODE_ENABLED']:
 
     # debug mode (flask reloader) runs two copies of the app. When running in debug mode,
     # we only want to start the scheduler one time.
-    if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
+    if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
       scheduler = BackgroundScheduler()
       heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
       heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
@@ -165,9 +172,9 @@ if app.config['HUB_MODE_ENABLED']:
     app.config['HUB_MODEL'] = hub_model.MockHub()
 
   from capsulflask import db
-  db.init_app(app)
+  db.init_app(app, is_running_server)
 
-  from capsulflask import auth, landing, console, payment, metrics, cli, hub_api
+  from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
 
   app.register_blueprint(landing.bp)
   app.register_blueprint(auth.bp)
@@ -176,6 +183,7 @@ if app.config['HUB_MODE_ENABLED']:
   app.register_blueprint(metrics.bp)
   app.register_blueprint(cli.bp)
   app.register_blueprint(hub_api.bp)
+  app.register_blueprint(admin.bp)
 
   app.add_url_rule("/", endpoint="index")
 

capsulflask/admin.py

@@ -3,16 +3,9 @@ import sys
 import json
 import ipaddress
 from datetime import datetime, timedelta
-from flask import Blueprint
+from flask import Blueprint, current_app, render_template, make_response
-from flask import flash
-from flask import current_app
-from flask import g
-from flask import request
-from flask import session
-from flask import render_template
-from flask import redirect
-from flask import url_for
 from werkzeug.exceptions import abort
+from nanoid import generate
 
 from capsulflask.metrics import durations as metric_durations
 from capsulflask.auth import admin_account_required
@@ -25,18 +18,23 @@ bp = Blueprint("admin", __name__, url_prefix="/admin")
 @admin_account_required
 def index():
   hosts = get_model().list_hosts_with_networks()
-  vms = get_model().all_non_deleted_vms()
+  vms_by_host_and_network = get_model().all_non_deleted_vms_by_host_and_network()
-  operations = get_model().list_all_operations()
+  network_display_width_px = float(250);
+  #operations = get_model().list_all_operations()
 
   display_hosts = []
+  inline_styles = [f"""
+  .network-display {'{'}
+    width: {network_display_width_px}px;
+  {'}'}
+  """]
 
   for kv in hosts.items():
-    name = kv[0]
+    host_id = kv[0]
     value = kv[1]
+    display_host = dict(name=host_id, networks=value['networks'])
 
-    for network in value['networks']:
+    for network in display_host['networks']:
-      network["network_name"]
-
       ipv4_network = ipaddress.ip_network(network["public_ipv4_cidr_block"], False)
       network_start_int = -1
       network_end_int = -1
@@ -49,9 +47,41 @@ def index():
           
           network_end_int = int(ipv4_address)
       
+      network['allocations'] = []
+      network_addresses_width = float((network_end_int-network_start_int)+1)
 
+      if host_id in vms_by_host_and_network:
+        if network['network_name'] in vms_by_host_and_network[host_id]:
+          for vm in vms_by_host_and_network[host_id][network['network_name']]:
+            ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
+            if network_start_int < ip_address_int and ip_address_int < network_end_int:
+              allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
+              inline_styles.append(
+                f"""
+                .{allocation} {'{'}
+                  left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px; 
+                  width: {network_display_width_px/network_addresses_width}px;
+                {'}'}
+                """
+              )
+              network['allocations'].append(allocation)
+            else:
+              current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
 
-    display_hosts.append(dict(name=name, last_health_check=value['last_health_check']))
+    display_hosts.append(display_host)
 
+  csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
+  response_text = render_template(
+    "admin.html", 
+    display_hosts=display_hosts, 
+    network_display_width_px=network_display_width_px,
+    csp_inline_style_nonce=csp_inline_style_nonce,
+    inline_style='\n'.join(inline_styles)
+  )
 
-  return render_template("admin.html", vms=mappedVms, has_vms=len(vms) > 0, created=created)
+  response = make_response(response_text)
+
+  response.headers.set('Content-Type', 'text/html')
+  response.headers.set('Content-Security-Policy', f"default-src 'self'; style-src 'self' 'nonce-{csp_inline_style_nonce}'")
+
+  return response

capsulflask/auth.py

@@ -40,7 +40,7 @@ def admin_account_required(view):
         if session.get("account") is None or session.get("csrf-token") is None:
             return redirect(url_for("auth.login"))
 
-        if session.get("account") not in current_app.config["ADMIN_EMAIL_ADDRESSES_CSV"].split(","):
+        if session.get("account") not in current_app.config["ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES"].split(","):
             return redirect(url_for("auth.login"))
 
         return view(**kwargs)

capsulflask/console.py

@@ -23,9 +23,9 @@ from capsulflask import cli
 
 bp = Blueprint("console", __name__, url_prefix="/console")
 
-def makeCapsulId():
+def make_capsul_id():
-  lettersAndNumbers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
+  letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
-  return f"capsul-{lettersAndNumbers}"
+  return f"capsul-{letters_n_nummers}"
 
 def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
   try:
@@ -244,7 +244,7 @@ def create():
       """)
 
     if len(errors) == 0:
-      id = makeCapsulId()
+      id = make_capsul_id()
       get_model().create_vm(
         email=session["account"], 
         id=id, 

capsulflask/db.py

@@ -10,7 +10,7 @@ from flask import g
 from capsulflask.db_model import DBModel
 from capsulflask.shared import my_exec_info_message
 
-def init_app(app):
+def init_app(app, is_running_server):
 
   app.config['PSYCOPG2_CONNECTION_POOL'] = psycopg2.pool.SimpleConnectionPool(
     1,
@@ -18,6 +18,14 @@ def init_app(app):
     app.config['POSTGRES_CONNECTION_PARAMETERS']
   )
 
+  # tell the app to clean up the DB connection when shutting down.
+  app.teardown_appcontext(close_db)
+
+  # only run the migrations if we are running the server. 
+  # If we are just running a cli command (e.g. to fix a broken migration 😅), skip it
+  if not is_running_server:
+    return
+
   schemaMigrations = {}
   schemaMigrationsPath = join(app.root_path, 'schema_migrations')
   app.logger.info("loading schema migration scripts from {}".format(schemaMigrationsPath))
@@ -35,7 +43,7 @@ def init_app(app):
   hasSchemaVersionTable = False
   actionWasTaken = False
   schemaVersion = 0
-  desiredSchemaVersion = 15
+  desiredSchemaVersion = 16
 
   cursor = connection.cursor()
 
@@ -103,7 +111,7 @@ def init_app(app):
     ("schema migration completed." if actionWasTaken else "schema is already up to date. "), schemaVersion
   ))
 
-  app.teardown_appcontext(close_db)
+  
 
 
 def get_model():

capsulflask/db_model.py

@@ -56,9 +56,23 @@ class DBModel:
 
   #     ------    VM & ACCOUNT MANAGEMENT     ---------  
 
-  def all_non_deleted_vms(self):
+  def all_non_deleted_vms_by_host_and_network(self):
-    self.cursor.execute("SELECT id, host, network_name, last_seen_ipv4, last_seen_ipv6 FROM vms WHERE deleted IS NULL")
+    self.cursor.execute("SELECT id, host, network_name, public_ipv4, public_ipv6 FROM vms WHERE deleted IS NULL")
-    return list(map(lambda x: dict(id=x[0], host=x[1], network_name=x[2], last_seen_ipv4=x[3], last_seen_ipv6=x[4]), self.cursor.fetchall()))
+
+    hosts = dict()
+    for row in self.cursor.fetchall():
+      host_id = row[1]
+      network_name = row[2]
+      if host_id not in hosts:
+        hosts[host_id] = dict()
+      if network_name not in hosts[host_id]:
+        hosts[host_id][network_name] = []
+      
+      hosts[host_id][network_name].append(
+        dict(id=row[0], public_ipv4=row[3], public_ipv6=row[4])
+      )
+
+    return hosts
 
   def all_non_deleted_vm_ids(self):
     self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
@@ -108,7 +122,7 @@ class DBModel:
 
   def list_vms_for_account(self, email):
     self.cursor.execute(""" 
-      SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
+      SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
         FROM vms JOIN vm_sizes on vms.size = vm_sizes.id
       WHERE vms.email = %s""", 
       (email, )
@@ -119,7 +133,7 @@ class DBModel:
     ))
 
   def update_vm_ip(self, email, id, ipv4):
-    self.cursor.execute("UPDATE vms SET last_seen_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
+    self.cursor.execute("UPDATE vms SET public_ipv4 = %s WHERE email = %s AND id = %s", (ipv4, email, id))
     self.connection.commit()
 
   def update_vm_ssh_host_keys(self, email, id, ssh_host_keys):
@@ -155,7 +169,7 @@ class DBModel:
 
   def get_vm_detail(self, email, id):
     self.cursor.execute(""" 
-      SELECT vms.id, vms.last_seen_ipv4, vms.last_seen_ipv6, os_images.description, vms.created, vms.deleted,
+      SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted,
             vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
       FROM vms 
         JOIN os_images on vms.os = os_images.id

capsulflask/schema_migrations/16_down_managed_ips.sql

@@ -1,6 +1,6 @@
 
 DROP TABLE host_network;
 
-ALTER TABLE vms DROP COLUMN network;
+ALTER TABLE vms DROP COLUMN network_name;
 
 UPDATE schemaversion SET version = 15;

capsulflask/schema_migrations/16_up_managed_ips.sql

@@ -1,14 +1,25 @@
 
 CREATE TABLE host_network (
-  public_ipv4_cidr_block  TEXT PRIMARY KEY NOT NULL,
+  public_ipv4_cidr_block  TEXT NOT NULL,
   network_name       TEXT NOT NULL,
   host               TEXT NOT NULL REFERENCES hosts(id) ON DELETE RESTRICT,
+  PRIMARY KEY (host, network_name)
 );
 
-INSERT INTO host_network (public_ipv4_cidr_block, network_name, host) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
-                                                                        ('baikal', 'virbr2', '69.61.2.194/26');
 
-ALTER TABLE vms ADD COLUMN network_name TEXT NOT NULL;
+INSERT INTO host_network (host, network_name, public_ipv4_cidr_block) VALUES ('baikal', 'virbr1', '69.61.2.162/27'),
+                                                                             ('baikal', 'virbr2', '69.61.2.194/26');
+ALTER TABLE vms RENAME COLUMN last_seen_ipv4 TO public_ipv4;
+ALTER TABLE vms RENAME COLUMN last_seen_ipv6 TO public_ipv6;
+ALTER TABLE vms ADD COLUMN network_name TEXT;
+
+UPDATE vms SET network_name = 'virbr1' WHERE public_ipv6 < '69.61.2.192';
+UPDATE vms SET network_name = 'virbr2' WHERE public_ipv6 >= '69.61.2.192';
+
+ALTER TABLE vms ALTER COLUMN network_name SET NOT NULL;
+
 ALTER TABLE vms ADD FOREIGN KEY (host, network_name) REFERENCES host_network(host, network_name) ON DELETE RESTRICT;
 
+
 UPDATE schemaversion SET version = 16;
+

capsulflask/static/style.css

@@ -356,3 +356,26 @@ footer {
   font-size: 0.8em;
 }
 
+.network-row {
+  background-color: #777e7350;
+  padding: 5px;
+  margin-top: 5px;
+}
+
+.network-display {
+  height: 1em;
+  border: 1px solid #777e73;
+}
+
+.network-display div {
+  position: absolute;
+}
+
+.network-display div div {
+  top: 1px;
+  height: calc(1em - 2px);
+  background-color: rgba(221, 169, 56, 0.8);
+  border: 1px solid rgba(255, 223, 155, 0.8);
+  box-sizing: border-box;
+  position: relative;
+}

capsulflask/templates/admin.html

@@ -0,0 +1,42 @@
+{% extends 'base.html' %}
+
+{% block title %}Capsul Admin{% endblock %}
+
+{% block content %}
+<style nonce="{{csp_inline_style_nonce}}">
+{{inline_style}}
+</style>
+<div class="row third-margin">
+  <h1>Capsul Admin</h1>
+</div>
+<div class="third-margin">
+  {% for display_host in display_hosts %}
+    <div class="row">
+      <h1>{{ display_host["name"] }}</h1>
+    </div>
+    {% for network in display_host["networks"] %}
+      <div class="row network-row">
+        <i>{{ network["network_name"] }}</i> 
+        <span>{{ network["public_ipv4_cidr_block"] }}</span>
+        <div class="network-display">
+          {% for allocation in network["allocations"] %}
+
+            {# This outer div is used as an abs position container & selected by CSS so don't remove it pls. #}
+            <div> 
+
+            <div class="{{allocation}}">
+            </div>
+
+            </div>
+
+          {% endfor %}
+        </div>
+      </div>
+    {% endfor %}
+    
+    <hr/>
+  {% endfor %}
+</div>
+{% endblock %}
+
+{% block pagesource %}/templates/admin.html{% endblock %}