autonomic-cooperative/capsul-flask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

trying to get admin tools in place

Browse Source
This commit is contained in:
forest 2021-12-09 14:38:17 -06:00
parent fd0d112834
commit 2ce638ab29
5 changed files with 201 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
capsulflask/admin.py
View File

@ -11,6 +11,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required from capsulflask.auth import admin_account_required
from capsulflask.db import get_model from capsulflask.db import get_model
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
from capsulflask.shared import my_exec_info_message from capsulflask.shared import my_exec_info_message
bp = Blueprint("admin", __name__, url_prefix="/admin") bp = Blueprint("admin", __name__, url_prefix="/admin")
@ -48,12 +49,13 @@ def index():
else: else:
return abort(400, "unknown form action") return abort(400, "unknown form action")
# moving on from the form post stuff...
# first create the hosts list w/ ip allocation visualization from the database # first create the hosts list w/ ip allocation visualization from the database
# #
db_hosts = get_model().list_hosts_with_networks(None) db_hosts = get_model().list_hosts_with_networks(None)
db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None) db_vms_by_id = get_all_vms_from_db()
network_display_width_px = float(270) network_display_width_px = float(270)
#operations = get_model().list_all_operations() #operations = get_model().list_all_operations()
@ -64,7 +66,13 @@ def index():
{'}'} {'}'}
"""] """]
db_vm_by_id = dict() db_vms_by_host_network = dict()
for vm in db_vms_by_id.values():
host_network_key = f"{vm['host']}_{vm['network_name']}"
if host_network_key not in db_vms_by_host_network:
db_vms_by_host_network[host_network_key] = []
db_vms_by_host_network[host_network_key].append(vm)
for kv in db_hosts.items(): for kv in db_hosts.items():
host_id = kv[0] host_id = kv[0]
@ -79,53 +87,63 @@ def index():
network['allocations'] = [] network['allocations'] = []
network_addresses_width = float((network_end_int-network_start_int)+1) network_addresses_width = float((network_end_int-network_start_int)+1)
if host_id in db_vms_by_host_and_network: host_network_key = f"{host_id}_{network['network_name']}"
if network['network_name'] in db_vms_by_host_and_network[host_id]: if host_network_key in db_vms_by_host_network:
for vm in db_vms_by_host_and_network[host_id][network['network_name']]: for vm in db_vms_by_host_network[host_network_key]:
vm['network_name'] = network['network_name'] ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
vm['virtual_bridge_name'] = network['virtual_bridge_name'] if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
vm['host'] = host_id allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
db_vm_by_id[vm['id']] = vm inline_styles.append(
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4'])) f"""
if network_start_int <= ip_address_int and ip_address_int <= network_end_int: .{allocation} {'{'}
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}" left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
inline_styles.append( width: {network_display_width_px/network_addresses_width}px;
f""" {'}'}
.{allocation} {'{'} """
left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px; )
width: {network_display_width_px/network_addresses_width}px; network['allocations'].append(allocation)
{'}'} else:
""" current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
)
network['allocations'].append(allocation)
else:
current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
display_hosts.append(display_host) display_hosts.append(display_host)
# Now creating the capsuls running status ui # Now creating the capsul consistency / running status ui
# #
virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network() virt_vms_by_id = get_all_vms_from_hosts()
# virt_vms_dict = dict() virt_vm_id_by_ipv4 = dict()
# for vm in virt_vms: for vm_id, virt_vm in virt_vms_by_id.items():
# virt_vms_dict[vm["id"]] = vm["state"] virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
# in_db_but_not_in_virt = [] db_vm_id_by_ipv4 = dict()
# needs_to_be_started = [] for vm_id, db_vm in db_vms_by_id.items():
# needs_to_be_started_missing_ipv4 = [] db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
has_wrong_ip.append(db_vm)
# for vm in db_vms:
# if vm["id"] not in virt_vms_dict:
# in_db_but_not_in_virt.append(vm["id"])
# elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
# if vm["id"] in db_vm_by_id:
# needs_to_be_started.append(db_vm_by_id[vm["id"]])
# else:
# needs_to_be_started_missing_ipv4.append(vm["id"])
# elif vm["ipv4"] != current_ipv4
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}") # current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
@ -142,8 +160,11 @@ def index():
csp_inline_style_nonce=csp_inline_style_nonce, csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles), inline_style='\n'.join(inline_styles),
db_vms_by_host_and_network=json.dumps(db_vms_by_host_and_network), in_db_but_not_in_virt=in_db_but_not_in_virt,
virt_vms_by_host_and_network=json.dumps(virt_vms_by_host_and_network), state_not_equal_to_desired_state=state_not_equal_to_desired_state,
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
has_wrong_ip=has_wrong_ip
) )
response = make_response(response_text) response = make_response(response_text)

82
capsulflask/consistency.py
View File

@ -10,7 +10,7 @@ from capsulflask.db import get_model
# "host": "baikal", # "host": "baikal",
# "network_name": "public1", # "network_name": "public1",
# "virtual_bridge_name": "virbr1", # "virtual_bridge_name": "virbr1",
# "state": "running" # "desired_state": "running"
# }, # },
# { ... }, # { ... },
# ... # ...
@ -41,8 +41,10 @@ def get_all_vms_from_db() -> dict:
return db_vms_by_id return db_vms_by_id
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
def get_all_vms_from_hosts() -> dict: def get_all_vms_from_hosts() -> dict:
virt_vms = current_app.config["HUB_MODEL"].get_all_by_host_and_network() virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
#virt_networks = current_app.config["HUB_MODEL"].virsh_netlist() #virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
db_hosts = get_model().list_hosts_with_networks(None) db_hosts = get_model().list_hosts_with_networks(None)
@ -52,57 +54,55 @@ def get_all_vms_from_hosts() -> dict:
host_id = kv[0] host_id = kv[0]
value = kv[1] value = kv[1]
for network in value['networks']: for network in value['networks']:
if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
vm['network_name'] = network['network_name']
vm['virtual_bridge_name'] = network['virtual_bridge_name']
vm['host'] = host_id
virt_vms_by_id[vm['id']] = vm
return virt_vms_by_id
for vm in db_vms:
if vm["id"] not in db_vms_by_id:
# TODO
raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
else:
db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
virt_vms = current_app.config["HUB_MODEL"].get_vm_()
def ensure_vms_and_db_are_synced(): def ensure_vms_and_db_are_synced():
pass
# Now creating the capsuls running status ui # # Now creating the capsuls running status ui
# # #
for vm in db_vms: # for vm in db_vms:
db_ids_dict[vm['id']] = vm['desired_state'] # db_ids_dict[vm['id']] = vm['desired_state']
for vm in virt_vms: # for vm in virt_vms:
virt_ids_dict[vm['id']] = vm['desired_state'] # virt_ids_dict[vm['id']] = vm['desired_state']
errors = list() # errors = list()
for id in db_ids_dict: # for id in db_ids_dict:
if id not in virt_ids_dict: # if id not in virt_ids_dict:
errors.append(f"{id} is in the database but not in the virtualization model") # errors.append(f"{id} is in the database but not in the virtualization model")
elif db_ids_dict[id] != virt_ids_dict[id]: # elif db_ids_dict[id] != virt_ids_dict[id]:
errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model") # errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
for id in virt_ids_dict: # for id in virt_ids_dict:
if id not in db_ids_dict: # if id not in db_ids_dict:
errors.append(f"{id} is in the virtualization model but not in the database") # errors.append(f"{id} is in the virtualization model but not in the database")
if len(errors) > 0: # if len(errors) > 0:
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") # email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) # email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") # current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
for error in errors: # for error in errors:
current_app.logger.info(f"cron_task: {error}.") # current_app.logger.info(f"cron_task: {error}.")
current_app.config["FLASK_MAIL_INSTANCE"].send( # current_app.config["FLASK_MAIL_INSTANCE"].send(
Message( # Message(
"Capsul Consistency Check Failed", # "Capsul Consistency Check Failed",
sender=current_app.config["MAIL_DEFAULT_SENDER"], # sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(errors), # body="\n".join(errors),
recipients=email_addresses # recipients=email_addresses
) # )
) # )

2
capsulflask/shell_scripts/virsh-net-list.sh
View File

@ -7,7 +7,7 @@ virsh net-list --all | tail -n +3 | awk '{ print $1 }' | while read -r network_n
if [ "$network_name" != "" ]; then if [ "$network_name" != "" ]; then
virtual_bridge_name="$(virsh net-info "$network_name" | grep -E '^Bridge:' | awk '{ print $2 }')" virtual_bridge_name="$(virsh net-info "$network_name" | grep -E '^Bridge:' | awk '{ print $2 }')"
capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')" capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')"
printf '%s\n {"name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name" printf '%s\n {"network_name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
delimiter="," delimiter=","
fi fi
done done

8
capsulflask/spoke_model.py
View File

@ -184,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
else: else:
current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'") current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network=network['name']) vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network_name=network['network_name'])
vms_by_id[vm['domain']]['macs'][mac] = True vms_by_id[vm['domain']]['macs'][mac] = True
@ -199,14 +199,12 @@ class ShellScriptSpoke(VirtualizationInterface):
for status in statuses: for status in statuses:
if status['mac-address'] in vm_id_by_mac: if status['mac-address'] in vm_id_by_mac:
vm_id = vm_id_by_mac[status['mac-address']] vm_id = vm_id_by_mac[status['mac-address']]
vms_by_id[vm_id]['ipv4'] = status['ip-address'] vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
else: else:
current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac") current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
networks = dict() networks = dict()
for vm_id in vms_by_id: for vm in vms_by_id.values():
vm = vms_by_id[vm_id]
if vm['network'] not in networks: if vm['network'] not in networks:
networks[vm['network']] = [] networks[vm['network']] = []

112
capsulflask/templates/admin.html
View File

@ -36,29 +36,98 @@
<hr/> <hr/>
{% endfor %} {% endfor %}
<div class="row">
<h1>db_vms_by_host_and_network</h1>
</div>
<div class="row">
<pre>
{{db_vms_by_host_and_network}}
</pre>
</div>
<div class="row">
<h1>virt_vms_by_host_and_network</h1>
</div>
<div class="row">
<pre>
{{virt_vms_by_host_and_network}}
</pre>
</div>
</div> </div>
{% if in_db_but_not_in_virt|length > 0 %}
<div class="third-margin">
<h1>🚨 in the database but not in the virtualization model 🚨</h1>
{% for vm in in_db_but_not_in_virt %}
<div class="row">
{{vm['id']}} {{vm['public_ipv4']}}
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if state_not_equal_to_desired_state|length > 0 %}
<div class="third-margin">
<h1>😴 vm state != desired state 😴</h1>
{% for vm in state_not_equal_to_desired_state %}
<div class="row">
<div>{{vm['id']}}: state={{vm['state']}} desired_state={{vm['desired_state']}}</div>
<form method="post">
<input type="hidden" name="action" value="set_state"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="state" value="{{vm['desired_state']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🚦 START/STOP"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if stole_someone_elses_ip_and_own_ip_avaliable|length > 0 %}
<div class="third-margin">
<h1>👻 stole someone elses ip and own desired ip is avaliable 👻</h1>
{% for vm in stole_someone_elses_ip_and_own_ip_avaliable %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="dhcp_reset"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🔨 DHCP RESET"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if has_wrong_ip|length > 0 %}
<div class="third-margin">
<h1>🥴 has wrong ip address 🥴</h1>
{% for vm in has_wrong_ip %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="dhcp_reset"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🔨 DHCP RESET"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
{% if stole_someone_elses_ip_but_own_ip_also_stolen|length > 0 %}
<div class="third-margin">
<h1>💀 stole someone elses ip but own desired ip was also stolen 💀</h1>
{% for vm in stole_someone_elses_ip_but_own_ip_also_stolen %}
<div class="row">
<div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
<form method="post">
<input type="hidden" name="action" value="stop_and_expire"></input>
<input type="hidden" name="id" value="{{vm['id']}}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<input type="submit" value="🛑 STOP AND EXPIRE DHCP LEASE"/>
</form>
</div>
{% endfor %}
<hr/>
</div>
{% endif %}
<div class="third-margin"> <div class="third-margin">
<div class="row"> <div class="row">
<h1>📢 Admin Megaphone: Email All Users With Active Capsuls 📢</h1> <h1>📢 Admin Megaphone: Email All Users With Active Capsuls 📢</h1>
@ -74,6 +143,7 @@
</div> </div>
</div> </div>
{% endblock %} {% endblock %}
{% block pagesource %}/templates/admin.html{% endblock %} {% block pagesource %}/templates/admin.html{% endblock %}