trying to get admin tools in place
This commit is contained in:
parent
fd0d112834
commit
2ce638ab29
@ -11,6 +11,7 @@ from nanoid import generate
|
||||
from capsulflask.metrics import durations as metric_durations
|
||||
from capsulflask.auth import admin_account_required
|
||||
from capsulflask.db import get_model
|
||||
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
|
||||
from capsulflask.shared import my_exec_info_message
|
||||
|
||||
bp = Blueprint("admin", __name__, url_prefix="/admin")
|
||||
@ -48,12 +49,13 @@ def index():
|
||||
else:
|
||||
return abort(400, "unknown form action")
|
||||
|
||||
# moving on from the form post stuff...
|
||||
|
||||
# first create the hosts list w/ ip allocation visualization from the database
|
||||
#
|
||||
|
||||
db_hosts = get_model().list_hosts_with_networks(None)
|
||||
db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
|
||||
db_vms_by_id = get_all_vms_from_db()
|
||||
network_display_width_px = float(270)
|
||||
#operations = get_model().list_all_operations()
|
||||
|
||||
@ -64,7 +66,13 @@ def index():
|
||||
{'}'}
|
||||
"""]
|
||||
|
||||
db_vm_by_id = dict()
|
||||
db_vms_by_host_network = dict()
|
||||
for vm in db_vms_by_id.values():
|
||||
host_network_key = f"{vm['host']}_{vm['network_name']}"
|
||||
if host_network_key not in db_vms_by_host_network:
|
||||
db_vms_by_host_network[host_network_key] = []
|
||||
db_vms_by_host_network[host_network_key].append(vm)
|
||||
|
||||
|
||||
for kv in db_hosts.items():
|
||||
host_id = kv[0]
|
||||
@ -79,53 +87,63 @@ def index():
|
||||
network['allocations'] = []
|
||||
network_addresses_width = float((network_end_int-network_start_int)+1)
|
||||
|
||||
if host_id in db_vms_by_host_and_network:
|
||||
if network['network_name'] in db_vms_by_host_and_network[host_id]:
|
||||
for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
|
||||
vm['network_name'] = network['network_name']
|
||||
vm['virtual_bridge_name'] = network['virtual_bridge_name']
|
||||
vm['host'] = host_id
|
||||
db_vm_by_id[vm['id']] = vm
|
||||
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
|
||||
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
|
||||
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
|
||||
inline_styles.append(
|
||||
f"""
|
||||
.{allocation} {'{'}
|
||||
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
|
||||
width: {network_display_width_px/network_addresses_width}px;
|
||||
{'}'}
|
||||
"""
|
||||
)
|
||||
network['allocations'].append(allocation)
|
||||
else:
|
||||
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
|
||||
host_network_key = f"{host_id}_{network['network_name']}"
|
||||
if host_network_key in db_vms_by_host_network:
|
||||
for vm in db_vms_by_host_network[host_network_key]:
|
||||
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
|
||||
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
|
||||
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
|
||||
inline_styles.append(
|
||||
f"""
|
||||
.{allocation} {'{'}
|
||||
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
|
||||
width: {network_display_width_px/network_addresses_width}px;
|
||||
{'}'}
|
||||
"""
|
||||
)
|
||||
network['allocations'].append(allocation)
|
||||
else:
|
||||
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
|
||||
|
||||
display_hosts.append(display_host)
|
||||
|
||||
|
||||
# Now creating the capsuls running status ui
|
||||
# Now creating the capsul consistency / running status ui
|
||||
#
|
||||
|
||||
virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
|
||||
virt_vms_by_id = get_all_vms_from_hosts()
|
||||
|
||||
# virt_vms_dict = dict()
|
||||
# for vm in virt_vms:
|
||||
# virt_vms_dict[vm["id"]] = vm["state"]
|
||||
virt_vm_id_by_ipv4 = dict()
|
||||
for vm_id, virt_vm in virt_vms_by_id.items():
|
||||
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
|
||||
|
||||
# in_db_but_not_in_virt = []
|
||||
# needs_to_be_started = []
|
||||
# needs_to_be_started_missing_ipv4 = []
|
||||
db_vm_id_by_ipv4 = dict()
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
|
||||
|
||||
in_db_but_not_in_virt = []
|
||||
state_not_equal_to_desired_state = []
|
||||
stole_someone_elses_ip_and_own_ip_avaliable = []
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen = []
|
||||
has_wrong_ip = []
|
||||
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
if vm_id not in virt_vms_by_id:
|
||||
in_db_but_not_in_virt.append(db_vm)
|
||||
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
|
||||
db_vm["state"] = virt_vms_by_id[vm_id]['state']
|
||||
state_not_equal_to_desired_state.append(db_vm)
|
||||
elif virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
|
||||
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
|
||||
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
|
||||
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
|
||||
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
|
||||
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
|
||||
else:
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
|
||||
|
||||
has_wrong_ip.append(db_vm)
|
||||
|
||||
# for vm in db_vms:
|
||||
# if vm["id"] not in virt_vms_dict:
|
||||
# in_db_but_not_in_virt.append(vm["id"])
|
||||
# elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
|
||||
# if vm["id"] in db_vm_by_id:
|
||||
# needs_to_be_started.append(db_vm_by_id[vm["id"]])
|
||||
# else:
|
||||
# needs_to_be_started_missing_ipv4.append(vm["id"])
|
||||
# elif vm["ipv4"] != current_ipv4
|
||||
|
||||
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
|
||||
|
||||
@ -142,8 +160,11 @@ def index():
|
||||
csp_inline_style_nonce=csp_inline_style_nonce,
|
||||
inline_style='\n'.join(inline_styles),
|
||||
|
||||
db_vms_by_host_and_network=json.dumps(db_vms_by_host_and_network),
|
||||
virt_vms_by_host_and_network=json.dumps(virt_vms_by_host_and_network),
|
||||
in_db_but_not_in_virt=in_db_but_not_in_virt,
|
||||
state_not_equal_to_desired_state=state_not_equal_to_desired_state,
|
||||
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
|
||||
has_wrong_ip=has_wrong_ip
|
||||
)
|
||||
|
||||
response = make_response(response_text)
|
||||
|
||||
@ -10,7 +10,7 @@ from capsulflask.db import get_model
|
||||
# "host": "baikal",
|
||||
# "network_name": "public1",
|
||||
# "virtual_bridge_name": "virbr1",
|
||||
# "state": "running"
|
||||
# "desired_state": "running"
|
||||
# },
|
||||
# { ... },
|
||||
# ...
|
||||
@ -41,8 +41,10 @@ def get_all_vms_from_db() -> dict:
|
||||
|
||||
return db_vms_by_id
|
||||
|
||||
|
||||
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
|
||||
def get_all_vms_from_hosts() -> dict:
|
||||
virt_vms = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
|
||||
virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
|
||||
#virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
|
||||
db_hosts = get_model().list_hosts_with_networks(None)
|
||||
|
||||
@ -52,57 +54,55 @@ def get_all_vms_from_hosts() -> dict:
|
||||
host_id = kv[0]
|
||||
value = kv[1]
|
||||
for network in value['networks']:
|
||||
if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
|
||||
for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
|
||||
vm['network_name'] = network['network_name']
|
||||
vm['virtual_bridge_name'] = network['virtual_bridge_name']
|
||||
vm['host'] = host_id
|
||||
virt_vms_by_id[vm['id']] = vm
|
||||
|
||||
|
||||
for vm in db_vms:
|
||||
if vm["id"] not in db_vms_by_id:
|
||||
# TODO
|
||||
raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
|
||||
else:
|
||||
db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
|
||||
|
||||
virt_vms = current_app.config["HUB_MODEL"].get_vm_()
|
||||
return virt_vms_by_id
|
||||
|
||||
def ensure_vms_and_db_are_synced():
|
||||
|
||||
|
||||
|
||||
# Now creating the capsuls running status ui
|
||||
#
|
||||
pass
|
||||
# # Now creating the capsuls running status ui
|
||||
# #
|
||||
|
||||
|
||||
|
||||
for vm in db_vms:
|
||||
db_ids_dict[vm['id']] = vm['desired_state']
|
||||
# for vm in db_vms:
|
||||
# db_ids_dict[vm['id']] = vm['desired_state']
|
||||
|
||||
for vm in virt_vms:
|
||||
virt_ids_dict[vm['id']] = vm['desired_state']
|
||||
# for vm in virt_vms:
|
||||
# virt_ids_dict[vm['id']] = vm['desired_state']
|
||||
|
||||
errors = list()
|
||||
# errors = list()
|
||||
|
||||
for id in db_ids_dict:
|
||||
if id not in virt_ids_dict:
|
||||
errors.append(f"{id} is in the database but not in the virtualization model")
|
||||
elif db_ids_dict[id] != virt_ids_dict[id]:
|
||||
errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
|
||||
# for id in db_ids_dict:
|
||||
# if id not in virt_ids_dict:
|
||||
# errors.append(f"{id} is in the database but not in the virtualization model")
|
||||
# elif db_ids_dict[id] != virt_ids_dict[id]:
|
||||
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
|
||||
|
||||
for id in virt_ids_dict:
|
||||
if id not in db_ids_dict:
|
||||
errors.append(f"{id} is in the virtualization model but not in the database")
|
||||
# for id in virt_ids_dict:
|
||||
# if id not in db_ids_dict:
|
||||
# errors.append(f"{id} is in the virtualization model but not in the database")
|
||||
|
||||
if len(errors) > 0:
|
||||
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
||||
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
||||
# if len(errors) > 0:
|
||||
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
||||
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
||||
|
||||
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
||||
for error in errors:
|
||||
current_app.logger.info(f"cron_task: {error}.")
|
||||
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
||||
# for error in errors:
|
||||
# current_app.logger.info(f"cron_task: {error}.")
|
||||
|
||||
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
||||
Message(
|
||||
"Capsul Consistency Check Failed",
|
||||
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
||||
body="\n".join(errors),
|
||||
recipients=email_addresses
|
||||
)
|
||||
)
|
||||
# current_app.config["FLASK_MAIL_INSTANCE"].send(
|
||||
# Message(
|
||||
# "Capsul Consistency Check Failed",
|
||||
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
||||
# body="\n".join(errors),
|
||||
# recipients=email_addresses
|
||||
# )
|
||||
# )
|
||||
@ -7,7 +7,7 @@ virsh net-list --all | tail -n +3 | awk '{ print $1 }' | while read -r network_n
|
||||
if [ "$network_name" != "" ]; then
|
||||
virtual_bridge_name="$(virsh net-info "$network_name" | grep -E '^Bridge:' | awk '{ print $2 }')"
|
||||
capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')"
|
||||
printf '%s\n {"name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
|
||||
printf '%s\n {"network_name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
|
||||
delimiter=","
|
||||
fi
|
||||
done
|
||||
|
||||
@ -184,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
|
||||
else:
|
||||
current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
|
||||
|
||||
vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network=network['name'])
|
||||
vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network_name=network['network_name'])
|
||||
|
||||
vms_by_id[vm['domain']]['macs'][mac] = True
|
||||
|
||||
@ -199,14 +199,12 @@ class ShellScriptSpoke(VirtualizationInterface):
|
||||
for status in statuses:
|
||||
if status['mac-address'] in vm_id_by_mac:
|
||||
vm_id = vm_id_by_mac[status['mac-address']]
|
||||
vms_by_id[vm_id]['ipv4'] = status['ip-address']
|
||||
vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
|
||||
else:
|
||||
current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
|
||||
|
||||
networks = dict()
|
||||
for vm_id in vms_by_id:
|
||||
vm = vms_by_id[vm_id]
|
||||
|
||||
for vm in vms_by_id.values():
|
||||
if vm['network'] not in networks:
|
||||
networks[vm['network']] = []
|
||||
|
||||
|
||||
@ -36,29 +36,98 @@
|
||||
|
||||
<hr/>
|
||||
{% endfor %}
|
||||
|
||||
<div class="row">
|
||||
<h1>db_vms_by_host_and_network</h1>
|
||||
</div>
|
||||
<div class="row">
|
||||
<pre>
|
||||
{{db_vms_by_host_and_network}}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div class="row">
|
||||
<h1>virt_vms_by_host_and_network</h1>
|
||||
</div>
|
||||
<div class="row">
|
||||
<pre>
|
||||
{{virt_vms_by_host_and_network}}
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
{% if in_db_but_not_in_virt|length > 0 %}
|
||||
<div class="third-margin">
|
||||
<h1>🚨 in the database but not in the virtualization model 🚨</h1>
|
||||
{% for vm in in_db_but_not_in_virt %}
|
||||
<div class="row">
|
||||
{{vm['id']}} {{vm['public_ipv4']}}
|
||||
</div>
|
||||
{% endfor %}
|
||||
<hr/>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
|
||||
{% if state_not_equal_to_desired_state|length > 0 %}
|
||||
<div class="third-margin">
|
||||
<h1>😴 vm state != desired state 😴</h1>
|
||||
{% for vm in state_not_equal_to_desired_state %}
|
||||
<div class="row">
|
||||
<div>{{vm['id']}}: state={{vm['state']}} desired_state={{vm['desired_state']}}</div>
|
||||
<form method="post">
|
||||
<input type="hidden" name="action" value="set_state"></input>
|
||||
<input type="hidden" name="id" value="{{vm['id']}}"></input>
|
||||
<input type="hidden" name="state" value="{{vm['desired_state']}}"></input>
|
||||
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
|
||||
<input type="submit" value="🚦 START/STOP"/>
|
||||
</form>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<hr/>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
|
||||
{% if stole_someone_elses_ip_and_own_ip_avaliable|length > 0 %}
|
||||
<div class="third-margin">
|
||||
<h1>👻 stole someone elses ip and own desired ip is avaliable 👻</h1>
|
||||
{% for vm in stole_someone_elses_ip_and_own_ip_avaliable %}
|
||||
<div class="row">
|
||||
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
|
||||
<form method="post">
|
||||
<input type="hidden" name="action" value="dhcp_reset"></input>
|
||||
<input type="hidden" name="id" value="{{vm['id']}}"></input>
|
||||
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
|
||||
<input type="submit" value="🔨 DHCP RESET"/>
|
||||
</form>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<hr/>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if has_wrong_ip|length > 0 %}
|
||||
<div class="third-margin">
|
||||
<h1>🥴 has wrong ip address 🥴</h1>
|
||||
{% for vm in has_wrong_ip %}
|
||||
<div class="row">
|
||||
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
|
||||
<form method="post">
|
||||
<input type="hidden" name="action" value="dhcp_reset"></input>
|
||||
<input type="hidden" name="id" value="{{vm['id']}}"></input>
|
||||
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
|
||||
<input type="submit" value="🔨 DHCP RESET"/>
|
||||
</form>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<hr/>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if stole_someone_elses_ip_but_own_ip_also_stolen|length > 0 %}
|
||||
<div class="third-margin">
|
||||
<h1>💀 stole someone elses ip but own desired ip was also stolen 💀</h1>
|
||||
{% for vm in stole_someone_elses_ip_but_own_ip_also_stolen %}
|
||||
<div class="row">
|
||||
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
|
||||
<form method="post">
|
||||
<input type="hidden" name="action" value="stop_and_expire"></input>
|
||||
<input type="hidden" name="id" value="{{vm['id']}}"></input>
|
||||
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
|
||||
<input type="submit" value="🛑 STOP AND EXPIRE DHCP LEASE"/>
|
||||
</form>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<hr/>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="third-margin">
|
||||
<div class="row">
|
||||
<h1>📢 Admin Megaphone: Email All Users With Active Capsuls 📢</h1>
|
||||
@ -74,6 +143,7 @@
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
{% endblock %}
|
||||
|
||||
{% block pagesource %}/templates/admin.html{% endblock %}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user