diff --git a/capsulflask/admin.py b/capsulflask/admin.py
index ffa7f58..ecdbdbb 100644
--- a/capsulflask/admin.py
+++ b/capsulflask/admin.py
@@ -11,6 +11,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required
from capsulflask.db import get_model
+from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
from capsulflask.shared import my_exec_info_message
bp = Blueprint("admin", __name__, url_prefix="/admin")
@@ -48,12 +49,13 @@ def index():
else:
return abort(400, "unknown form action")
+ # moving on from the form post stuff...
# first create the hosts list w/ ip allocation visualization from the database
#
db_hosts = get_model().list_hosts_with_networks(None)
- db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
+ db_vms_by_id = get_all_vms_from_db()
network_display_width_px = float(270)
#operations = get_model().list_all_operations()
@@ -64,7 +66,13 @@ def index():
{'}'}
"""]
- db_vm_by_id = dict()
+ db_vms_by_host_network = dict()
+ for vm in db_vms_by_id.values():
+ host_network_key = f"{vm['host']}_{vm['network_name']}"
+ if host_network_key not in db_vms_by_host_network:
+ db_vms_by_host_network[host_network_key] = []
+ db_vms_by_host_network[host_network_key].append(vm)
+
for kv in db_hosts.items():
host_id = kv[0]
@@ -79,53 +87,63 @@ def index():
network['allocations'] = []
network_addresses_width = float((network_end_int-network_start_int)+1)
- if host_id in db_vms_by_host_and_network:
- if network['network_name'] in db_vms_by_host_and_network[host_id]:
- for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
- vm['network_name'] = network['network_name']
- vm['virtual_bridge_name'] = network['virtual_bridge_name']
- vm['host'] = host_id
- db_vm_by_id[vm['id']] = vm
- ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
- if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
- allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
- inline_styles.append(
- f"""
- .{allocation} {'{'}
- left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
- width: {network_display_width_px/network_addresses_width}px;
- {'}'}
- """
- )
- network['allocations'].append(allocation)
- else:
- current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
-
+ host_network_key = f"{host_id}_{network['network_name']}"
+ if host_network_key in db_vms_by_host_network:
+ for vm in db_vms_by_host_network[host_network_key]:
+ ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
+ if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
+ allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
+ inline_styles.append(
+ f"""
+ .{allocation} {'{'}
+ left: {(float(ip_address_int-network_start_int)/network_addresses_width)*network_display_width_px}px;
+ width: {network_display_width_px/network_addresses_width}px;
+ {'}'}
+ """
+ )
+ network['allocations'].append(allocation)
+ else:
+ current_app.logger.warning(f"/admin: capsul {vm['id']} has public_ipv4 {vm['public_ipv4']} which is out of range for its host network {host_id} {network['network_name']} {network['public_ipv4_cidr_block']}")
+
display_hosts.append(display_host)
- # Now creating the capsuls running status ui
+ # Now creating the capsul consistency / running status ui
#
- virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
-
- # virt_vms_dict = dict()
- # for vm in virt_vms:
- # virt_vms_dict[vm["id"]] = vm["state"]
+ virt_vms_by_id = get_all_vms_from_hosts()
- # in_db_but_not_in_virt = []
- # needs_to_be_started = []
- # needs_to_be_started_missing_ipv4 = []
+ virt_vm_id_by_ipv4 = dict()
+ for vm_id, virt_vm in virt_vms_by_id.items():
+ virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
+
+ db_vm_id_by_ipv4 = dict()
+ for vm_id, db_vm in db_vms_by_id.items():
+ db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
+
+ in_db_but_not_in_virt = []
+ state_not_equal_to_desired_state = []
+ stole_someone_elses_ip_and_own_ip_avaliable = []
+ stole_someone_elses_ip_but_own_ip_also_stolen = []
+ has_wrong_ip = []
+
+ for vm_id, db_vm in db_vms_by_id.items():
+ if vm_id not in virt_vms_by_id:
+ in_db_but_not_in_virt.append(db_vm)
+ elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
+ db_vm["state"] = virt_vms_by_id[vm_id]['state']
+ state_not_equal_to_desired_state.append(db_vm)
+ elif virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
+ db_vm["desired_ipv4"] = db_vm["public_ipv4"]
+ db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
+ if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
+ if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
+ stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
+ else:
+ stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
+
+ has_wrong_ip.append(db_vm)
- # for vm in db_vms:
- # if vm["id"] not in virt_vms_dict:
- # in_db_but_not_in_virt.append(vm["id"])
- # elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
- # if vm["id"] in db_vm_by_id:
- # needs_to_be_started.append(db_vm_by_id[vm["id"]])
- # else:
- # needs_to_be_started_missing_ipv4.append(vm["id"])
- # elif vm["ipv4"] != current_ipv4
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
@@ -142,8 +160,11 @@ def index():
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles),
- db_vms_by_host_and_network=json.dumps(db_vms_by_host_and_network),
- virt_vms_by_host_and_network=json.dumps(virt_vms_by_host_and_network),
+ in_db_but_not_in_virt=in_db_but_not_in_virt,
+ state_not_equal_to_desired_state=state_not_equal_to_desired_state,
+ stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
+ stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
+ has_wrong_ip=has_wrong_ip
)
response = make_response(response_text)
diff --git a/capsulflask/consistency.py b/capsulflask/consistency.py
index bb379e9..d28513f 100644
--- a/capsulflask/consistency.py
+++ b/capsulflask/consistency.py
@@ -10,7 +10,7 @@ from capsulflask.db import get_model
# "host": "baikal",
# "network_name": "public1",
# "virtual_bridge_name": "virbr1",
-# "state": "running"
+# "desired_state": "running"
# },
# { ... },
# ...
@@ -41,8 +41,10 @@ def get_all_vms_from_db() -> dict:
return db_vms_by_id
+
+# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
def get_all_vms_from_hosts() -> dict:
- virt_vms = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
+ virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
#virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
db_hosts = get_model().list_hosts_with_networks(None)
@@ -52,57 +54,55 @@ def get_all_vms_from_hosts() -> dict:
host_id = kv[0]
value = kv[1]
for network in value['networks']:
-
+ if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
+ for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
+ vm['network_name'] = network['network_name']
+ vm['virtual_bridge_name'] = network['virtual_bridge_name']
+ vm['host'] = host_id
+ virt_vms_by_id[vm['id']] = vm
- for vm in db_vms:
- if vm["id"] not in db_vms_by_id:
- # TODO
- raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
- else:
- db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
-
- virt_vms = current_app.config["HUB_MODEL"].get_vm_()
+ return virt_vms_by_id
def ensure_vms_and_db_are_synced():
-
- # Now creating the capsuls running status ui
- #
+ pass
+ # # Now creating the capsuls running status ui
+ # #
- for vm in db_vms:
- db_ids_dict[vm['id']] = vm['desired_state']
+ # for vm in db_vms:
+ # db_ids_dict[vm['id']] = vm['desired_state']
- for vm in virt_vms:
- virt_ids_dict[vm['id']] = vm['desired_state']
+ # for vm in virt_vms:
+ # virt_ids_dict[vm['id']] = vm['desired_state']
- errors = list()
+ # errors = list()
- for id in db_ids_dict:
- if id not in virt_ids_dict:
- errors.append(f"{id} is in the database but not in the virtualization model")
- elif db_ids_dict[id] != virt_ids_dict[id]:
- errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
+ # for id in db_ids_dict:
+ # if id not in virt_ids_dict:
+ # errors.append(f"{id} is in the database but not in the virtualization model")
+ # elif db_ids_dict[id] != virt_ids_dict[id]:
+ # errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
- for id in virt_ids_dict:
- if id not in db_ids_dict:
- errors.append(f"{id} is in the virtualization model but not in the database")
+ # for id in virt_ids_dict:
+ # if id not in db_ids_dict:
+ # errors.append(f"{id} is in the virtualization model but not in the database")
- if len(errors) > 0:
- email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
- email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
+ # if len(errors) > 0:
+ # email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
+ # email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
- current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
- for error in errors:
- current_app.logger.info(f"cron_task: {error}.")
+ # current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
+ # for error in errors:
+ # current_app.logger.info(f"cron_task: {error}.")
- current_app.config["FLASK_MAIL_INSTANCE"].send(
- Message(
- "Capsul Consistency Check Failed",
- sender=current_app.config["MAIL_DEFAULT_SENDER"],
- body="\n".join(errors),
- recipients=email_addresses
- )
- )
\ No newline at end of file
+ # current_app.config["FLASK_MAIL_INSTANCE"].send(
+ # Message(
+ # "Capsul Consistency Check Failed",
+ # sender=current_app.config["MAIL_DEFAULT_SENDER"],
+ # body="\n".join(errors),
+ # recipients=email_addresses
+ # )
+ # )
\ No newline at end of file
diff --git a/capsulflask/shell_scripts/virsh-net-list.sh b/capsulflask/shell_scripts/virsh-net-list.sh
index 67967d2..912dc2e 100755
--- a/capsulflask/shell_scripts/virsh-net-list.sh
+++ b/capsulflask/shell_scripts/virsh-net-list.sh
@@ -7,7 +7,7 @@ virsh net-list --all | tail -n +3 | awk '{ print $1 }' | while read -r network_n
if [ "$network_name" != "" ]; then
virtual_bridge_name="$(virsh net-info "$network_name" | grep -E '^Bridge:' | awk '{ print $2 }')"
capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')"
- printf '%s\n {"name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
+ printf '%s\n {"network_name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
delimiter=","
fi
done
diff --git a/capsulflask/spoke_model.py b/capsulflask/spoke_model.py
index b455ece..1afa0a2 100644
--- a/capsulflask/spoke_model.py
+++ b/capsulflask/spoke_model.py
@@ -184,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
else:
current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
- vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network=network['name'])
+ vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network_name=network['network_name'])
vms_by_id[vm['domain']]['macs'][mac] = True
@@ -199,14 +199,12 @@ class ShellScriptSpoke(VirtualizationInterface):
for status in statuses:
if status['mac-address'] in vm_id_by_mac:
vm_id = vm_id_by_mac[status['mac-address']]
- vms_by_id[vm_id]['ipv4'] = status['ip-address']
+ vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
else:
current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
networks = dict()
- for vm_id in vms_by_id:
- vm = vms_by_id[vm_id]
-
+ for vm in vms_by_id.values():
if vm['network'] not in networks:
networks[vm['network']] = []
diff --git a/capsulflask/templates/admin.html b/capsulflask/templates/admin.html
index 7b6d236..e0a62bb 100644
--- a/capsulflask/templates/admin.html
+++ b/capsulflask/templates/admin.html
@@ -36,29 +36,98 @@
{% endfor %}
-
-
-
db_vms_by_host_and_network
-
-
-
-{{db_vms_by_host_and_network}}
-
-
-
virt_vms_by_host_and_network
-
-
-
-{{virt_vms_by_host_and_network}}
-
-
+
🚨 in the database but not in the virtualization model 🚨
+ {% for vm in in_db_but_not_in_virt %}
+
+ {{vm['id']}} {{vm['public_ipv4']}}
+
+ {% endfor %}
+
+
+
😴 vm state != desired state 😴
+ {% for vm in state_not_equal_to_desired_state %}
+
+
{{vm['id']}}: state={{vm['state']}} desired_state={{vm['desired_state']}}
+
+
+ {% endfor %}
+
+
+
👻 stole someone elses ip and own desired ip is avaliable 👻
+ {% for vm in stole_someone_elses_ip_and_own_ip_avaliable %}
+
+
{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}
+
+
+ {% endfor %}
+
+
+
🥴 has wrong ip address 🥴
+ {% for vm in has_wrong_ip %}
+
+
{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}
+
+
+ {% endfor %}
+
+
+
💀 stole someone elses ip but own desired ip was also stolen 💀
+ {% for vm in stole_someone_elses_ip_but_own_ip_also_stolen %}
+
+
{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}
+
+
+ {% endfor %}
+
+
📢 Admin Megaphone: Email All Users With Active Capsuls 📢
@@ -74,6 +143,7 @@
+
{% endblock %}
{% block pagesource %}/templates/admin.html{% endblock %}