autonomic-cooperative/capsul-flask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

crontask: monitor db vs virt model of all capsuls like admin panel

Browse Source
This commit is contained in:
forest 2021-12-17 18:29:48 -06:00
parent 5637da73ec
commit b013f9c975
3 changed files with 86 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
capsulflask/admin.py
View File

@ -12,7 +12,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required from capsulflask.auth import admin_account_required
from capsulflask.db import get_model from capsulflask.db import get_model
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
from capsulflask.shared import my_exec_info_message from capsulflask.shared import my_exec_info_message
bp = Blueprint("admin", __name__, url_prefix="/admin") bp = Blueprint("admin", __name__, url_prefix="/admin")
@ -181,50 +181,7 @@ def index():
display_hosts.append(display_host) display_hosts.append(display_host)
# Now creating the capsul consistency / running status ui inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
#
virt_vm_id_by_ipv4 = dict()
for vm_id, virt_vm in virt_vms_by_id.items():
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
db_vm_id_by_ipv4 = dict()
for vm_id, db_vm in db_vms_by_id.items():
if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
has_no_desired_ip_address = []
has_not_aquired_ip_address_yet = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
has_no_desired_ip_address.append(db_vm)
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
has_not_aquired_ip_address_yet.append(db_vm)
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
has_wrong_ip.append(db_vm)
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
@ -235,13 +192,13 @@ def index():
network_display_width_px=network_display_width_px, network_display_width_px=network_display_width_px,
csp_inline_style_nonce=csp_inline_style_nonce, csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles), inline_style='\n'.join(inline_styles),
in_db_but_not_in_virt=in_db_but_not_in_virt, in_db_but_not_in_virt=inconsistency_info['in_db_but_not_in_virt'],
state_not_equal_to_desired_state=state_not_equal_to_desired_state, state_not_equal_to_desired_state=inconsistency_info['state_not_equal_to_desired_state'],
has_no_desired_ip_address=has_no_desired_ip_address, has_no_desired_ip_address=inconsistency_info['has_no_desired_ip_address'],
has_not_aquired_ip_address_yet=has_not_aquired_ip_address_yet, has_not_aquired_ip_address_yet=inconsistency_info['has_not_aquired_ip_address_yet'],
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable, stole_someone_elses_ip_and_own_ip_avaliable=inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable'],
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen, stole_someone_elses_ip_but_own_ip_also_stolen=inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen'],
has_wrong_ip=has_wrong_ip has_wrong_ip=inconsistency_info['has_wrong_ip']
) )
response = make_response(response_text) response = make_response(response_text)

62
capsulflask/cli.py
View File

@ -14,6 +14,7 @@ from flask_mail import Message
from capsulflask.db import get_model from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message from capsulflask.shared import my_exec_info_message
from capsulflask.console import get_account_balance from capsulflask.console import get_account_balance
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
bp = Blueprint('cli', __name__) bp = Blueprint('cli', __name__)
@ -268,46 +269,41 @@ def notify_users_about_account_balance():
def ensure_vms_and_db_are_synced(): def ensure_vms_and_db_are_synced():
pass db_vms_by_id = get_all_vms_from_db()
# db_vms = get_model().all_vm_ids_with_desired_state() virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)
# # TODO replaced inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
# #virt_vms = current_app.config["HUB_MODEL"].virsh_list()
# db_ids_dict = dict() errors = list()
# virt_ids_dict = dict()
# for vm in db_vms: for vm in inconsistency_info['in_db_but_not_in_virt']:
# db_ids_dict[vm['id']] = vm['desired_state'] errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model")
# for vm in virt_vms: for vm in inconsistency_info['state_not_equal_to_desired_state']:
# virt_ids_dict[vm['id']] = vm['desired_state'] errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}")
# errors = list() for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']:
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
# for id in db_ids_dict: for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']:
# if id not in virt_ids_dict: errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
# errors.append(f"{id} is in the database but not in the virtualization model")
# elif db_ids_dict[id] != virt_ids_dict[id]:
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
# for id in virt_ids_dict: for vm in inconsistency_info['has_wrong_ip']:
# if id not in db_ids_dict: errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
# errors.append(f"{id} is in the virtualization model but not in the database")
# if len(errors) > 0: if len(errors) > 0:
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
# for error in errors: for error in errors:
# current_app.logger.info(f"cron_task: {error}.") current_app.logger.info(f"cron_task: {error}.")
# current_app.config["FLASK_MAIL_INSTANCE"].send( current_app.config["FLASK_MAIL_INSTANCE"].send(
# Message( Message(
# "Capsul Consistency Check Failed", "Capsul Consistency Check Failed",
# sender=current_app.config["MAIL_DEFAULT_SENDER"], sender=current_app.config["MAIL_DEFAULT_SENDER"],
# body="\n".join(errors), body="\n".join(errors),
# recipients=email_addresses recipients=email_addresses
# ) )
# ) )

93
capsulflask/consistency.py
View File

@ -33,17 +33,11 @@ def get_all_vms_from_db() -> dict:
vm['host'] = host_id vm['host'] = host_id
db_vms_by_id[vm['id']] = vm db_vms_by_id[vm['id']] = vm
# for vm in db_vms:
# if vm["id"] not in db_vms_by_id:
# # TODO
# raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
# else:
# db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
return db_vms_by_id return db_vms_by_id
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state' # this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'.
# also it has `macs` which is a list of strings
def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict: def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id() virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id()
#current_app.logger.info(f"\n*******************2:\n{pprint.pformat(virt_vms_by_id)}\n\n\n\n") #current_app.logger.info(f"\n*******************2:\n{pprint.pformat(virt_vms_by_id)}\n\n\n\n")
@ -68,46 +62,53 @@ def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
return virt_vms_by_id return virt_vms_by_id
def ensure_vms_and_db_are_synced(): def get_inconsistent_capsuls_information(db_vms_by_id: dict, virt_vms_by_id: dict) -> dict:
virt_vm_id_by_ipv4 = dict()
for vm_id, virt_vm in virt_vms_by_id.items():
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
pass db_vm_id_by_ipv4 = dict()
# # Now creating the capsuls running status ui for vm_id, db_vm in db_vms_by_id.items():
# # if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
has_no_desired_ip_address = []
has_not_aquired_ip_address_yet = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
has_no_desired_ip_address.append(db_vm)
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
has_not_aquired_ip_address_yet.append(db_vm)
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
else:
has_wrong_ip.append(db_vm)
# for vm in db_vms: return {
# db_ids_dict[vm['id']] = vm['desired_state'] "in_db_but_not_in_virt": in_db_but_not_in_virt,
"state_not_equal_to_desired_state": state_not_equal_to_desired_state,
# for vm in virt_vms: "has_no_desired_ip_address": has_no_desired_ip_address,
# virt_ids_dict[vm['id']] = vm['desired_state'] "has_not_aquired_ip_address_yet": has_not_aquired_ip_address_yet,
"stole_someone_elses_ip_and_own_ip_avaliable": stole_someone_elses_ip_and_own_ip_avaliable,
# errors = list() "stole_someone_elses_ip_but_own_ip_also_stolen": stole_someone_elses_ip_but_own_ip_also_stolen,
"has_wrong_ip": has_wrong_ip,
# for id in db_ids_dict: }
# if id not in virt_ids_dict:
# errors.append(f"{id} is in the database but not in the virtualization model")
# elif db_ids_dict[id] != virt_ids_dict[id]:
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
# for id in virt_ids_dict:
# if id not in db_ids_dict:
# errors.append(f"{id} is in the virtualization model but not in the database")
# if len(errors) > 0:
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
# for error in errors:
# current_app.logger.info(f"cron_task: {error}.")
# current_app.config["FLASK_MAIL_INSTANCE"].send(
# Message(
# "Capsul Consistency Check Failed",
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
# body="\n".join(errors),
# recipients=email_addresses
# )
# )