trying to get admin tools in place

This commit is contained in:
forest 2021-12-09 14:38:17 -06:00
parent fd0d112834
commit 2ce638ab29
5 changed files with 201 additions and 112 deletions

View File

@ -11,6 +11,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required
from capsulflask.db import get_model
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
from capsulflask.shared import my_exec_info_message
bp = Blueprint("admin", __name__, url_prefix="/admin")
@ -48,12 +49,13 @@ def index():
else:
return abort(400, "unknown form action")
# moving on from the form post stuff...
# first create the hosts list w/ ip allocation visualization from the database
#
db_hosts = get_model().list_hosts_with_networks(None)
db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
db_vms_by_id = get_all_vms_from_db()
network_display_width_px = float(270)
#operations = get_model().list_all_operations()
@ -64,7 +66,13 @@ def index():
{'}'}
"""]
db_vm_by_id = dict()
db_vms_by_host_network = dict()
for vm in db_vms_by_id.values():
host_network_key = f"{vm['host']}_{vm['network_name']}"
if host_network_key not in db_vms_by_host_network:
db_vms_by_host_network[host_network_key] = []
db_vms_by_host_network[host_network_key].append(vm)
for kv in db_hosts.items():
host_id = kv[0]
@ -79,53 +87,63 @@ def index():
network['allocations'] = []
network_addresses_width = float((network_end_int-network_start_int)+1)
if host_id in db_vms_by_host_and_network:
if network['network_name'] in db_vms_by_host_and_network[host_id]:
for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
vm['network_name'] = network['network_name']
vm['virtual_bridge_name'] = network['virtual_bridge_name']
vm['host'] = host_id
db_vm_by_id[vm['id']] = vm
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
inline_styles.append(
f"""
.{allocation} {'{'}
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
width: {network_display_width_px/network_addresses_width}px;
{'}'}
"""
)
network['allocations'].append(allocation)
else:
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
host_network_key = f"{host_id}_{network['network_name']}"
if host_network_key in db_vms_by_host_network:
for vm in db_vms_by_host_network[host_network_key]:
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
inline_styles.append(
f"""
.{allocation} {'{'}
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
width: {network_display_width_px/network_addresses_width}px;
{'}'}
"""
)
network['allocations'].append(allocation)
else:
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
display_hosts.append(display_host)
# Now creating the capsuls running status ui
# Now creating the capsul consistency / running status ui
#
virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
virt_vms_by_id = get_all_vms_from_hosts()
# virt_vms_dict = dict()
# for vm in virt_vms:
# virt_vms_dict[vm["id"]] = vm["state"]
virt_vm_id_by_ipv4 = dict()
for vm_id, virt_vm in virt_vms_by_id.items():
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
# in_db_but_not_in_virt = []
# needs_to_be_started = []
# needs_to_be_started_missing_ipv4 = []
db_vm_id_by_ipv4 = dict()
for vm_id, db_vm in db_vms_by_id.items():
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
has_wrong_ip.append(db_vm)
# for vm in db_vms:
# if vm["id"] not in virt_vms_dict:
# in_db_but_not_in_virt.append(vm["id"])
# elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
# if vm["id"] in db_vm_by_id:
# needs_to_be_started.append(db_vm_by_id[vm["id"]])
# else:
# needs_to_be_started_missing_ipv4.append(vm["id"])
# elif vm["ipv4"] != current_ipv4
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
@ -142,8 +160,11 @@ def index():
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles),
db_vms_by_host_and_network=json.dumps(db_vms_by_host_and_network),
virt_vms_by_host_and_network=json.dumps(virt_vms_by_host_and_network),
in_db_but_not_in_virt=in_db_but_not_in_virt,
state_not_equal_to_desired_state=state_not_equal_to_desired_state,
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
has_wrong_ip=has_wrong_ip
)
response = make_response(response_text)

View File

@ -10,7 +10,7 @@ from capsulflask.db import get_model
# "host": "baikal",
# "network_name": "public1",
# "virtual_bridge_name": "virbr1",
# "state": "running"
# "desired_state": "running"
# },
# { ... },
# ...
@ -41,8 +41,10 @@ def get_all_vms_from_db() -> dict:
return db_vms_by_id
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
def get_all_vms_from_hosts() -> dict:
virt_vms = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
#virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
db_hosts = get_model().list_hosts_with_networks(None)
@ -52,57 +54,55 @@ def get_all_vms_from_hosts() -> dict:
host_id = kv[0]
value = kv[1]
for network in value['networks']:
if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
vm['network_name'] = network['network_name']
vm['virtual_bridge_name'] = network['virtual_bridge_name']
vm['host'] = host_id
virt_vms_by_id[vm['id']] = vm
for vm in db_vms:
if vm["id"] not in db_vms_by_id:
# TODO
raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
else:
db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
virt_vms = current_app.config["HUB_MODEL"].get_vm_()
return virt_vms_by_id
def ensure_vms_and_db_are_synced():
# Now creating the capsuls running status ui
#
pass
# # Now creating the capsuls running status ui
# #
for vm in db_vms:
db_ids_dict[vm['id']] = vm['desired_state']
# for vm in db_vms:
# db_ids_dict[vm['id']] = vm['desired_state']
for vm in virt_vms:
virt_ids_dict[vm['id']] = vm['desired_state']
# for vm in virt_vms:
# virt_ids_dict[vm['id']] = vm['desired_state']
errors = list()
# errors = list()
for id in db_ids_dict:
if id not in virt_ids_dict:
errors.append(f"{id} is in the database but not in the virtualization model")
elif db_ids_dict[id] != virt_ids_dict[id]:
errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
# for id in db_ids_dict:
# if id not in virt_ids_dict:
# errors.append(f"{id} is in the database but not in the virtualization model")
# elif db_ids_dict[id] != virt_ids_dict[id]:
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
for id in virt_ids_dict:
if id not in db_ids_dict:
errors.append(f"{id} is in the virtualization model but not in the database")
# for id in virt_ids_dict:
# if id not in db_ids_dict:
# errors.append(f"{id} is in the virtualization model but not in the database")
if len(errors) > 0:
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
# if len(errors) > 0:
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
for error in errors:
current_app.logger.info(f"cron_task: {error}.")
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
# for error in errors:
# current_app.logger.info(f"cron_task: {error}.")
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Capsul Consistency Check Failed",
sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(errors),
recipients=email_addresses
)
)
# current_app.config["FLASK_MAIL_INSTANCE"].send(
# Message(
# "Capsul Consistency Check Failed",
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
# body="\n".join(errors),
# recipients=email_addresses
# )
# )

View File

@ -7,7 +7,7 @@ virsh net-list --all | tail -n +3 | awk '{ print $1 }' | while read -r network_n
if [ "$network_name" != "" ]; then
virtual_bridge_name="$(virsh net-info "$network_name" | grep -E '^Bridge:' | awk '{ print $2 }')"
capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')"
printf '%s\n {"name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
printf '%s\n {"network_name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
delimiter=","
fi
done

View File

@ -184,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
else:
current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network=network['name'])
vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network_name=network['network_name'])
vms_by_id[vm['domain']]['macs'][mac] = True
@ -199,14 +199,12 @@ class ShellScriptSpoke(VirtualizationInterface):
for status in statuses:
if status['mac-address'] in vm_id_by_mac:
vm_id = vm_id_by_mac[status['mac-address']]
vms_by_id[vm_id]['ipv4'] = status['ip-address']
vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
else:
current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
networks = dict()
for vm_id in vms_by_id:
vm = vms_by_id[vm_id]
for vm in vms_by_id.values():
if vm['network'] not in networks:
networks[vm['network']] = []

View File

@ -36,29 +36,98 @@
<hr/>
{% endfor %}
<div class="row">
<h1>db_vms_by_host_and_network</h1>
</div>
<div class="row">
<pre>
{{db_vms_by_host_and_network}}
</pre>
</div>
<div class="row">
<h1>virt_vms_by_host_and_network</h1>
</div>
<div class="row">
<pre>
{{virt_vms_by_host_and_network}}
</pre>
</div>
</div>
{% if in_db_but_not_in_virt|length > 0 %}
<div class="third-margin">
<h1>🚨 in the database but not in the virtualization model 🚨</h1>
{% for vm in in_db_but_not_in_virt %}
<div class="row">
{{vm['id']}} {{vm['public_ipv4']}}
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if state_not_equal_to_desired_state|length > 0 %}
<div class="third-margin">
<h1>😴 vm state != desired state 😴</h1>
{% for vm in state_not_equal_to_desired_state %}
<div class="row">
<div>{{vm['id']}}: state={{vm['state']}} desired_state={{vm['desired_state']}}</div>
<form method="post">
<input type="hidden" name="action" value="set_state"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="state" value="{{vm['desired_state']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🚦 START/STOP"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if stole_someone_elses_ip_and_own_ip_avaliable|length > 0 %}
<div class="third-margin">
<h1>👻 stole someone elses ip and own desired ip is avaliable 👻</h1>
{% for vm in stole_someone_elses_ip_and_own_ip_avaliable %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="dhcp_reset"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🔨 DHCP RESET"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if has_wrong_ip|length > 0 %}
<div class="third-margin">
<h1>🥴 has wrong ip address 🥴</h1>
{% for vm in has_wrong_ip %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="dhcp_reset"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🔨 DHCP RESET"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if stole_someone_elses_ip_but_own_ip_also_stolen|length > 0 %}
<div class="third-margin">
<h1>💀 stole someone elses ip but own desired ip was also stolen 💀</h1>
{% for vm in stole_someone_elses_ip_but_own_ip_also_stolen %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="stop_and_expire"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🛑 STOP AND EXPIRE DHCP LEASE"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
<div class="third-margin">
<div class="row">
<h1>📢 Admin Megaphone: Email All Users With Active Capsuls 📢</h1>
@ -74,6 +143,7 @@
</div>
</div>
{% endblock %}
{% block pagesource %}/templates/admin.html{% endblock %}