crontask: monitor db vs virt model of all capsuls like admin panel
This commit is contained in:
parent
5637da73ec
commit
b013f9c975
@ -12,7 +12,7 @@ from nanoid import generate
|
||||
from capsulflask.metrics import durations as metric_durations
|
||||
from capsulflask.auth import admin_account_required
|
||||
from capsulflask.db import get_model
|
||||
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
|
||||
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
|
||||
from capsulflask.shared import my_exec_info_message
|
||||
|
||||
bp = Blueprint("admin", __name__, url_prefix="/admin")
|
||||
@ -181,50 +181,7 @@ def index():
|
||||
display_hosts.append(display_host)
|
||||
|
||||
|
||||
# Now creating the capsul consistency / running status ui
|
||||
#
|
||||
|
||||
virt_vm_id_by_ipv4 = dict()
|
||||
for vm_id, virt_vm in virt_vms_by_id.items():
|
||||
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
|
||||
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
|
||||
|
||||
db_vm_id_by_ipv4 = dict()
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
|
||||
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
|
||||
|
||||
in_db_but_not_in_virt = []
|
||||
state_not_equal_to_desired_state = []
|
||||
has_no_desired_ip_address = []
|
||||
has_not_aquired_ip_address_yet = []
|
||||
stole_someone_elses_ip_and_own_ip_avaliable = []
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen = []
|
||||
has_wrong_ip = []
|
||||
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
if vm_id not in virt_vms_by_id:
|
||||
in_db_but_not_in_virt.append(db_vm)
|
||||
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
|
||||
db_vm["state"] = virt_vms_by_id[vm_id]['state']
|
||||
state_not_equal_to_desired_state.append(db_vm)
|
||||
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
|
||||
has_no_desired_ip_address.append(db_vm)
|
||||
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
|
||||
has_not_aquired_ip_address_yet.append(db_vm)
|
||||
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
|
||||
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
|
||||
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
|
||||
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
|
||||
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
|
||||
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
|
||||
else:
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
|
||||
|
||||
has_wrong_ip.append(db_vm)
|
||||
|
||||
|
||||
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
|
||||
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
|
||||
|
||||
|
||||
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
|
||||
@ -235,13 +192,13 @@ def index():
|
||||
network_display_width_px=network_display_width_px,
|
||||
csp_inline_style_nonce=csp_inline_style_nonce,
|
||||
inline_style='\n'.join(inline_styles),
|
||||
in_db_but_not_in_virt=in_db_but_not_in_virt,
|
||||
state_not_equal_to_desired_state=state_not_equal_to_desired_state,
|
||||
has_no_desired_ip_address=has_no_desired_ip_address,
|
||||
has_not_aquired_ip_address_yet=has_not_aquired_ip_address_yet,
|
||||
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
|
||||
has_wrong_ip=has_wrong_ip
|
||||
in_db_but_not_in_virt=inconsistency_info['in_db_but_not_in_virt'],
|
||||
state_not_equal_to_desired_state=inconsistency_info['state_not_equal_to_desired_state'],
|
||||
has_no_desired_ip_address=inconsistency_info['has_no_desired_ip_address'],
|
||||
has_not_aquired_ip_address_yet=inconsistency_info['has_not_aquired_ip_address_yet'],
|
||||
stole_someone_elses_ip_and_own_ip_avaliable=inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable'],
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen=inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen'],
|
||||
has_wrong_ip=inconsistency_info['has_wrong_ip']
|
||||
)
|
||||
|
||||
response = make_response(response_text)
|
||||
|
||||
@ -14,6 +14,7 @@ from flask_mail import Message
|
||||
from capsulflask.db import get_model
|
||||
from capsulflask.shared import my_exec_info_message
|
||||
from capsulflask.console import get_account_balance
|
||||
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
|
||||
|
||||
bp = Blueprint('cli', __name__)
|
||||
|
||||
@ -268,46 +269,41 @@ def notify_users_about_account_balance():
|
||||
|
||||
|
||||
def ensure_vms_and_db_are_synced():
|
||||
pass
|
||||
# db_vms = get_model().all_vm_ids_with_desired_state()
|
||||
# # TODO replaced
|
||||
# #virt_vms = current_app.config["HUB_MODEL"].virsh_list()
|
||||
db_vms_by_id = get_all_vms_from_db()
|
||||
virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)
|
||||
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
|
||||
|
||||
# db_ids_dict = dict()
|
||||
# virt_ids_dict = dict()
|
||||
errors = list()
|
||||
|
||||
# for vm in db_vms:
|
||||
# db_ids_dict[vm['id']] = vm['desired_state']
|
||||
|
||||
# for vm in virt_vms:
|
||||
# virt_ids_dict[vm['id']] = vm['desired_state']
|
||||
for vm in inconsistency_info['in_db_but_not_in_virt']:
|
||||
errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model")
|
||||
|
||||
# errors = list()
|
||||
for vm in inconsistency_info['state_not_equal_to_desired_state']:
|
||||
errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}")
|
||||
|
||||
# for id in db_ids_dict:
|
||||
# if id not in virt_ids_dict:
|
||||
# errors.append(f"{id} is in the database but not in the virtualization model")
|
||||
# elif db_ids_dict[id] != virt_ids_dict[id]:
|
||||
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
|
||||
for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']:
|
||||
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
||||
|
||||
# for id in virt_ids_dict:
|
||||
# if id not in db_ids_dict:
|
||||
# errors.append(f"{id} is in the virtualization model but not in the database")
|
||||
for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']:
|
||||
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
||||
|
||||
# if len(errors) > 0:
|
||||
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
||||
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
||||
for vm in inconsistency_info['has_wrong_ip']:
|
||||
errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
|
||||
|
||||
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
||||
# for error in errors:
|
||||
# current_app.logger.info(f"cron_task: {error}.")
|
||||
if len(errors) > 0:
|
||||
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
||||
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
||||
|
||||
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
||||
for error in errors:
|
||||
current_app.logger.info(f"cron_task: {error}.")
|
||||
|
||||
# current_app.config["FLASK_MAIL_INSTANCE"].send(
|
||||
# Message(
|
||||
# "Capsul Consistency Check Failed",
|
||||
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
||||
# body="\n".join(errors),
|
||||
# recipients=email_addresses
|
||||
# )
|
||||
# )
|
||||
current_app.config["FLASK_MAIL_INSTANCE"].send(
|
||||
Message(
|
||||
"Capsul Consistency Check Failed",
|
||||
sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
||||
body="\n".join(errors),
|
||||
recipients=email_addresses
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@ -32,18 +32,12 @@ def get_all_vms_from_db() -> dict:
|
||||
vm['virtual_bridge_name'] = network['virtual_bridge_name']
|
||||
vm['host'] = host_id
|
||||
db_vms_by_id[vm['id']] = vm
|
||||
|
||||
# for vm in db_vms:
|
||||
# if vm["id"] not in db_vms_by_id:
|
||||
# # TODO
|
||||
# raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
|
||||
# else:
|
||||
# db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
|
||||
|
||||
return db_vms_by_id
|
||||
|
||||
|
||||
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
|
||||
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'.
|
||||
# also it has `macs` which is a list of strings
|
||||
def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
|
||||
virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id()
|
||||
#current_app.logger.info(f"\n*******************2:\n{pprint.pformat(virt_vms_by_id)}\n\n\n\n")
|
||||
@ -68,46 +62,53 @@ def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
|
||||
|
||||
return virt_vms_by_id
|
||||
|
||||
def ensure_vms_and_db_are_synced():
|
||||
|
||||
|
||||
pass
|
||||
# # Now creating the capsuls running status ui
|
||||
# #
|
||||
def get_inconsistent_capsuls_information(db_vms_by_id: dict, virt_vms_by_id: dict) -> dict:
|
||||
|
||||
virt_vm_id_by_ipv4 = dict()
|
||||
for vm_id, virt_vm in virt_vms_by_id.items():
|
||||
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
|
||||
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
|
||||
|
||||
db_vm_id_by_ipv4 = dict()
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
|
||||
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
|
||||
|
||||
# for vm in db_vms:
|
||||
# db_ids_dict[vm['id']] = vm['desired_state']
|
||||
|
||||
# for vm in virt_vms:
|
||||
# virt_ids_dict[vm['id']] = vm['desired_state']
|
||||
in_db_but_not_in_virt = []
|
||||
state_not_equal_to_desired_state = []
|
||||
has_no_desired_ip_address = []
|
||||
has_not_aquired_ip_address_yet = []
|
||||
stole_someone_elses_ip_and_own_ip_avaliable = []
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen = []
|
||||
has_wrong_ip = []
|
||||
|
||||
# errors = list()
|
||||
for vm_id, db_vm in db_vms_by_id.items():
|
||||
if vm_id not in virt_vms_by_id:
|
||||
in_db_but_not_in_virt.append(db_vm)
|
||||
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
|
||||
db_vm["state"] = virt_vms_by_id[vm_id]['state']
|
||||
state_not_equal_to_desired_state.append(db_vm)
|
||||
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
|
||||
has_no_desired_ip_address.append(db_vm)
|
||||
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
|
||||
has_not_aquired_ip_address_yet.append(db_vm)
|
||||
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
|
||||
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
|
||||
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
|
||||
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
|
||||
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
|
||||
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
|
||||
else:
|
||||
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
|
||||
else:
|
||||
has_wrong_ip.append(db_vm)
|
||||
|
||||
# for id in db_ids_dict:
|
||||
# if id not in virt_ids_dict:
|
||||
# errors.append(f"{id} is in the database but not in the virtualization model")
|
||||
# elif db_ids_dict[id] != virt_ids_dict[id]:
|
||||
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
|
||||
|
||||
# for id in virt_ids_dict:
|
||||
# if id not in db_ids_dict:
|
||||
# errors.append(f"{id} is in the virtualization model but not in the database")
|
||||
|
||||
# if len(errors) > 0:
|
||||
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
|
||||
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
|
||||
|
||||
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
|
||||
# for error in errors:
|
||||
# current_app.logger.info(f"cron_task: {error}.")
|
||||
|
||||
# current_app.config["FLASK_MAIL_INSTANCE"].send(
|
||||
# Message(
|
||||
# "Capsul Consistency Check Failed",
|
||||
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
|
||||
# body="\n".join(errors),
|
||||
# recipients=email_addresses
|
||||
# )
|
||||
# )
|
||||
return {
|
||||
"in_db_but_not_in_virt": in_db_but_not_in_virt,
|
||||
"state_not_equal_to_desired_state": state_not_equal_to_desired_state,
|
||||
"has_no_desired_ip_address": has_no_desired_ip_address,
|
||||
"has_not_aquired_ip_address_yet": has_not_aquired_ip_address_yet,
|
||||
"stole_someone_elses_ip_and_own_ip_avaliable": stole_someone_elses_ip_and_own_ip_avaliable,
|
||||
"stole_someone_elses_ip_but_own_ip_also_stolen": stole_someone_elses_ip_but_own_ip_also_stolen,
|
||||
"has_wrong_ip": has_wrong_ip,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user