From b013f9c9758f2cc062f1ecefc4d7deef3aa484f2 Mon Sep 17 00:00:00 2001 From: forest Date: Fri, 17 Dec 2021 18:29:48 -0600 Subject: [PATCH] crontask: monitor db vs virt model of all capsuls like admin panel --- capsulflask/admin.py | 61 ++++--------------------- capsulflask/cli.py | 64 ++++++++++++-------------- capsulflask/consistency.py | 93 +++++++++++++++++++------------------- 3 files changed, 86 insertions(+), 132 deletions(-) diff --git a/capsulflask/admin.py b/capsulflask/admin.py index 12c80c7..6b57197 100644 --- a/capsulflask/admin.py +++ b/capsulflask/admin.py @@ -12,7 +12,7 @@ from nanoid import generate from capsulflask.metrics import durations as metric_durations from capsulflask.auth import admin_account_required from capsulflask.db import get_model -from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts +from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information from capsulflask.shared import my_exec_info_message bp = Blueprint("admin", __name__, url_prefix="/admin") @@ -181,50 +181,7 @@ def index(): display_hosts.append(display_host) - # Now creating the capsul consistency / running status ui - # - - virt_vm_id_by_ipv4 = dict() - for vm_id, virt_vm in virt_vms_by_id.items(): - if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "": - virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id - - db_vm_id_by_ipv4 = dict() - for vm_id, db_vm in db_vms_by_id.items(): - if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "": - db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id - - in_db_but_not_in_virt = [] - state_not_equal_to_desired_state = [] - has_no_desired_ip_address = [] - has_not_aquired_ip_address_yet = [] - stole_someone_elses_ip_and_own_ip_avaliable = [] - stole_someone_elses_ip_but_own_ip_also_stolen = [] - has_wrong_ip = [] - - for vm_id, db_vm in db_vms_by_id.items(): - if vm_id not in virt_vms_by_id: - in_db_but_not_in_virt.append(db_vm) - elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]: - db_vm["state"] = virt_vms_by_id[vm_id]['state'] - state_not_equal_to_desired_state.append(db_vm) - elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "": - has_no_desired_ip_address.append(db_vm) - elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""): - has_not_aquired_ip_address_yet.append(db_vm) - elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]: - db_vm["desired_ipv4"] = db_vm["public_ipv4"] - db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4'] - if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4: - if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4: - stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm) - else: - stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm) - - has_wrong_ip.append(db_vm) - - - # current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}") + inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id) csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) @@ -235,13 +192,13 @@ def index(): network_display_width_px=network_display_width_px, csp_inline_style_nonce=csp_inline_style_nonce, inline_style='\n'.join(inline_styles), - in_db_but_not_in_virt=in_db_but_not_in_virt, - state_not_equal_to_desired_state=state_not_equal_to_desired_state, - has_no_desired_ip_address=has_no_desired_ip_address, - has_not_aquired_ip_address_yet=has_not_aquired_ip_address_yet, - stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable, - stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen, - has_wrong_ip=has_wrong_ip + in_db_but_not_in_virt=inconsistency_info['in_db_but_not_in_virt'], + state_not_equal_to_desired_state=inconsistency_info['state_not_equal_to_desired_state'], + has_no_desired_ip_address=inconsistency_info['has_no_desired_ip_address'], + has_not_aquired_ip_address_yet=inconsistency_info['has_not_aquired_ip_address_yet'], + stole_someone_elses_ip_and_own_ip_avaliable=inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable'], + stole_someone_elses_ip_but_own_ip_also_stolen=inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen'], + has_wrong_ip=inconsistency_info['has_wrong_ip'] ) response = make_response(response_text) diff --git a/capsulflask/cli.py b/capsulflask/cli.py index 3c83e47..cc7af18 100644 --- a/capsulflask/cli.py +++ b/capsulflask/cli.py @@ -14,6 +14,7 @@ from flask_mail import Message from capsulflask.db import get_model from capsulflask.shared import my_exec_info_message from capsulflask.console import get_account_balance +from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information bp = Blueprint('cli', __name__) @@ -268,46 +269,41 @@ def notify_users_about_account_balance(): def ensure_vms_and_db_are_synced(): - pass - # db_vms = get_model().all_vm_ids_with_desired_state() - # # TODO replaced - # #virt_vms = current_app.config["HUB_MODEL"].virsh_list() + db_vms_by_id = get_all_vms_from_db() + virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id) + inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id) - # db_ids_dict = dict() - # virt_ids_dict = dict() + errors = list() - # for vm in db_vms: - # db_ids_dict[vm['id']] = vm['desired_state'] - - # for vm in virt_vms: - # virt_ids_dict[vm['id']] = vm['desired_state'] + for vm in inconsistency_info['in_db_but_not_in_virt']: + errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model") - # errors = list() + for vm in inconsistency_info['state_not_equal_to_desired_state']: + errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}") - # for id in db_ids_dict: - # if id not in virt_ids_dict: - # errors.append(f"{id} is in the database but not in the virtualization model") - # elif db_ids_dict[id] != virt_ids_dict[id]: - # errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model") + for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']: + errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") - # for id in virt_ids_dict: - # if id not in db_ids_dict: - # errors.append(f"{id} is in the virtualization model but not in the database") + for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']: + errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") - # if len(errors) > 0: - # email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") - # email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) + for vm in inconsistency_info['has_wrong_ip']: + errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}") - # current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") - # for error in errors: - # current_app.logger.info(f"cron_task: {error}.") + if len(errors) > 0: + email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") + email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) + + current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") + for error in errors: + current_app.logger.info(f"cron_task: {error}.") - # current_app.config["FLASK_MAIL_INSTANCE"].send( - # Message( - # "Capsul Consistency Check Failed", - # sender=current_app.config["MAIL_DEFAULT_SENDER"], - # body="\n".join(errors), - # recipients=email_addresses - # ) - # ) + current_app.config["FLASK_MAIL_INSTANCE"].send( + Message( + "Capsul Consistency Check Failed", + sender=current_app.config["MAIL_DEFAULT_SENDER"], + body="\n".join(errors), + recipients=email_addresses + ) + ) diff --git a/capsulflask/consistency.py b/capsulflask/consistency.py index 33c4f3f..9d635d3 100644 --- a/capsulflask/consistency.py +++ b/capsulflask/consistency.py @@ -32,18 +32,12 @@ def get_all_vms_from_db() -> dict: vm['virtual_bridge_name'] = network['virtual_bridge_name'] vm['host'] = host_id db_vms_by_id[vm['id']] = vm - - # for vm in db_vms: - # if vm["id"] not in db_vms_by_id: - # # TODO - # raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") - # else: - # db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] return db_vms_by_id -# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state' +# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'. +# also it has `macs` which is a list of strings def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict: virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id() #current_app.logger.info(f"\n*******************2:\n{pprint.pformat(virt_vms_by_id)}\n\n\n\n") @@ -68,46 +62,53 @@ def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict: return virt_vms_by_id -def ensure_vms_and_db_are_synced(): - - - pass - # # Now creating the capsuls running status ui - # # +def get_inconsistent_capsuls_information(db_vms_by_id: dict, virt_vms_by_id: dict) -> dict: + virt_vm_id_by_ipv4 = dict() + for vm_id, virt_vm in virt_vms_by_id.items(): + if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "": + virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id + db_vm_id_by_ipv4 = dict() + for vm_id, db_vm in db_vms_by_id.items(): + if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "": + db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id - # for vm in db_vms: - # db_ids_dict[vm['id']] = vm['desired_state'] - - # for vm in virt_vms: - # virt_ids_dict[vm['id']] = vm['desired_state'] + in_db_but_not_in_virt = [] + state_not_equal_to_desired_state = [] + has_no_desired_ip_address = [] + has_not_aquired_ip_address_yet = [] + stole_someone_elses_ip_and_own_ip_avaliable = [] + stole_someone_elses_ip_but_own_ip_also_stolen = [] + has_wrong_ip = [] - # errors = list() + for vm_id, db_vm in db_vms_by_id.items(): + if vm_id not in virt_vms_by_id: + in_db_but_not_in_virt.append(db_vm) + elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]: + db_vm["state"] = virt_vms_by_id[vm_id]['state'] + state_not_equal_to_desired_state.append(db_vm) + elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "": + has_no_desired_ip_address.append(db_vm) + elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""): + has_not_aquired_ip_address_yet.append(db_vm) + elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]: + db_vm["desired_ipv4"] = db_vm["public_ipv4"] + db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4'] + if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4: + if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4: + stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm) + else: + stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm) + else: + has_wrong_ip.append(db_vm) - # for id in db_ids_dict: - # if id not in virt_ids_dict: - # errors.append(f"{id} is in the database but not in the virtualization model") - # elif db_ids_dict[id] != virt_ids_dict[id]: - # errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model") - - # for id in virt_ids_dict: - # if id not in db_ids_dict: - # errors.append(f"{id} is in the virtualization model but not in the database") - - # if len(errors) > 0: - # email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") - # email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) - - # current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") - # for error in errors: - # current_app.logger.info(f"cron_task: {error}.") - - # current_app.config["FLASK_MAIL_INSTANCE"].send( - # Message( - # "Capsul Consistency Check Failed", - # sender=current_app.config["MAIL_DEFAULT_SENDER"], - # body="\n".join(errors), - # recipients=email_addresses - # ) - # ) \ No newline at end of file + return { + "in_db_but_not_in_virt": in_db_but_not_in_virt, + "state_not_equal_to_desired_state": state_not_equal_to_desired_state, + "has_no_desired_ip_address": has_no_desired_ip_address, + "has_not_aquired_ip_address_yet": has_not_aquired_ip_address_yet, + "stole_someone_elses_ip_and_own_ip_avaliable": stole_someone_elses_ip_and_own_ip_avaliable, + "stole_someone_elses_ip_but_own_ip_also_stolen": stole_someone_elses_ip_but_own_ip_also_stolen, + "has_wrong_ip": has_wrong_ip, + }