From 4bbacffeae6bbab47bb694850654f12c6f2503ac Mon Sep 17 00:00:00 2001 From: forest Date: Fri, 10 Dec 2021 19:53:14 -0500 Subject: [PATCH] refactor get_all_by_host_and_network to get_all_by_id --- capsulflask/consistency.py | 33 +++++++++++++++------------- capsulflask/db_model.py | 2 +- capsulflask/hub_model.py | 26 +++++++++++----------- capsulflask/shared.py | 2 +- capsulflask/spoke_api.py | 6 +++--- capsulflask/spoke_model.py | 44 ++++++++++++++++---------------------- 6 files changed, 55 insertions(+), 58 deletions(-) diff --git a/capsulflask/consistency.py b/capsulflask/consistency.py index d28513f..873bbe7 100644 --- a/capsulflask/consistency.py +++ b/capsulflask/consistency.py @@ -43,23 +43,26 @@ def get_all_vms_from_db() -> dict: # this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state' -def get_all_vms_from_hosts() -> dict: - virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network() - #virt_networks = current_app.config["HUB_MODEL"].virsh_netlist() +def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict: + virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id() db_hosts = get_model().list_hosts_with_networks(None) - - virt_vms_by_id = dict() - for kv in db_hosts.items(): - host_id = kv[0] - value = kv[1] - for network in value['networks']: - if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]: - for vm in virt_vms_by_host_and_network[host_id][network['network_name']]: - vm['network_name'] = network['network_name'] - vm['virtual_bridge_name'] = network['virtual_bridge_name'] - vm['host'] = host_id - virt_vms_by_id[vm['id']] = vm + for vm_id, virt_vm in virt_vms_by_id.items(): + if vm_id in db_vms_by_id: + if 'host' not in virt_vm: + virt_vm['host'] = db_vms_by_id[vm_id]['host'] + if 'network_name' not in virt_vm: + virt_vm['network_name'] = db_vms_by_id[vm_id]['network_name'] + else: + if 'host' not in virt_vm: + virt_vm['host'] = "unknown" + if 'network_name' not in virt_vm: + virt_vm['network_name'] = "unknown" + + if virt_vm['host'] in db_hosts and virt_vm['network_name'] in db_hosts[virt_vm['host']]: + virt_vm['virtual_bridge_name'] = db_hosts[virt_vm['host']][virt_vm['network_name']]['virtual_bridge_name'] + else: + virt_vm['virtual_bridge_name'] = "unknown" return virt_vms_by_id diff --git a/capsulflask/db_model.py b/capsulflask/db_model.py index a28ac3c..9fd0fe9 100644 --- a/capsulflask/db_model.py +++ b/capsulflask/db_model.py @@ -81,7 +81,7 @@ class DBModel: hosts[host_id][network_name] = [] hosts[host_id][network_name].append( - dict(id=row[0], email=row[1], desired_state=row[2], public_ipv4=row[5], public_ipv6=row[6]) + dict(id=row[0], email=row[1], host=host_id, network_name=network_name, desired_state=row[2], public_ipv4=row[5], public_ipv6=row[6]) ) return hosts diff --git a/capsulflask/hub_model.py b/capsulflask/hub_model.py index 07279dc..8ee78e8 100644 --- a/capsulflask/hub_model.py +++ b/capsulflask/hub_model.py @@ -37,8 +37,15 @@ class MockHub(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4) - def get_all_by_host_and_network(self) -> dict: - return get_model().non_deleted_vms_by_host_and_network() + def get_all_by_id(self) -> dict: + by_host_and_network = get_model().non_deleted_vms_by_host_and_network() + to_return = dict() + for host in by_host_and_network.values(): + for network in host.values(): + for vm in network: + to_return[vm['id']] = vm + + return to_return def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) @@ -167,9 +174,9 @@ class CapsulFlaskHub(VirtualizationInterface): return None - def get_all_by_host_and_network(self) -> dict: + def get_all_by_id(self) -> dict: online_hosts = get_model().get_online_hosts() - payload = json.dumps(dict(type="get_all_by_host_and_network")) + payload = json.dumps(dict(type="get_all_by_id")) results = self.synchronous_operation(online_hosts, None, payload) to_return = dict() for i in range(len(results)): @@ -177,14 +184,9 @@ class CapsulFlaskHub(VirtualizationInterface): result = results[i] try: result_body = json.loads(result.body) - - has_hosts = isinstance(result_body, dict) and "hosts" in result_body and isinstance(result_body["hosts"], dict) - has_current_host = has_hosts and isinstance(result_body["hosts"], dict) and host.id in result_body["hosts"] and isinstance(result_body["hosts"][host.id], dict) - if has_current_host: - to_return[host.id] = result_body["hosts"][host.id] - else: - # result_json_string = json.dumps({"error_message": "invalid response, missing 'networks' list"}) - current_app.logger.error(f"""missing 'hosts' dict for get_all_by_host_and_network operation, host {host.id}""") + for id, vm in result_body.items(): + vm['host'] = host.id + to_return[id] = vm except: # no need to do anything here since if it cant be parsed then generic_operation will handle it. pass diff --git a/capsulflask/shared.py b/capsulflask/shared.py index 2fa4f12..77476d3 100644 --- a/capsulflask/shared.py +++ b/capsulflask/shared.py @@ -32,7 +32,7 @@ class VirtualizationInterface: def get(self, id: str) -> VirtualMachine: pass - def get_all_by_host_and_network(self) -> dict: + def get_all_by_id(self) -> dict: pass def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list): diff --git a/capsulflask/spoke_api.py b/capsulflask/spoke_api.py index 09b2d6d..7ae80ed 100644 --- a/capsulflask/spoke_api.py +++ b/capsulflask/spoke_api.py @@ -49,7 +49,7 @@ def operation_impl(operation_id: int): handlers = { "capacity_avaliable": handle_capacity_avaliable, "get": handle_get, - "get_all_by_host_and_network": handle_get_all_by_host_and_network, + "get_all_by_id": handle_get_all_by_id, "create": handle_create, "destroy": handle_destroy, "vm_state_command": handle_vm_state_command, @@ -97,8 +97,8 @@ def handle_get(operation_id, request_body): return jsonify(dict(assignment_status="assigned", id=vm.id, host=vm.host, state=vm.state, ipv4=vm.ipv4, ipv6=vm.ipv6, ssh_host_keys=vm.ssh_host_keys)) -def handle_get_all_by_host_and_network(operation_id, request_body) -> dict: - return jsonify(dict(assignment_status="assigned", hosts=current_app.config['SPOKE_MODEL'].get_all_by_host_and_network())) +def handle_get_all_by_id(operation_id, request_body) -> dict: + return jsonify(dict(assignment_status="assigned", vms=current_app.config['SPOKE_MODEL'].get_all_by_id())) def handle_create(operation_id, request_body): if not operation_id: diff --git a/capsulflask/spoke_model.py b/capsulflask/spoke_model.py index d0a25a1..41f91d6 100644 --- a/capsulflask/spoke_model.py +++ b/capsulflask/spoke_model.py @@ -39,9 +39,15 @@ class MockSpoke(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running") - def get_all_by_host_and_network(self) -> dict: - to_return = get_model().non_deleted_vms_by_host_and_network(None) - current_app.logger.info(f"MOCK get_all_by_host_and_network: {json.dumps(to_return)}") + def get_all_by_id(self) -> dict: + by_host_and_network = get_model().non_deleted_vms_by_host_and_network() + to_return = dict() + for host in by_host_and_network.values(): + for network in host.values(): + for vm in network: + to_return[vm['id']] = vm + + current_app.logger.info(f"MOCK get_all_by_id: {json.dumps(to_return)}") return to_return def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str): @@ -140,7 +146,7 @@ class ShellScriptSpoke(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr) - def get_all_by_host_and_network(self) -> dict: + def get_all_by_id(self) -> dict: vm_list_process = run([join(current_app.root_path, 'shell_scripts/virsh-list.sh')], capture_output=True) self.validate_completed_process(vm_list_process) @@ -150,9 +156,9 @@ class ShellScriptSpoke(VirtualizationInterface): #current_app.logger.info(f"list_of_vms: {json.dumps(list_of_vms)}") - vm_state_by_id = dict() + vms_by_id = dict() for vm in list_of_vms: - vm_state_by_id[vm['id']] = vm['state'] + vms_by_id[vm['id']] = dict(id=vm['id'], macs=dict(), state=vm['state']) net_list_process = run([join(current_app.root_path, 'shell_scripts/virsh-net-list.sh')], capture_output=True) self.validate_completed_process(net_list_process) @@ -162,7 +168,6 @@ class ShellScriptSpoke(VirtualizationInterface): #current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}") - vms_by_id = dict() vm_id_by_mac = dict() for network in list_of_networks: @@ -182,14 +187,11 @@ class ShellScriptSpoke(VirtualizationInterface): raise Exception(f"the mac address '{mac}' is used by both '{vm_id_by_mac[mac]}' and '{vm['domain']}'") if vm['domain'] not in vms_by_id: - vm_state = 'shut off' - if vm['domain'] in vm_state_by_id: - vm_state = vm_state_by_id[vm['domain']] - else: - current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'") + current_app.logger.warn(f"'{vm['domain']}' was in dnsmask but not in libvirt, defaulting to 'shut off' state") - vms_by_id[vm['domain']] = dict(id=vm['domain'], macs=dict(), state=vm_state, network_name=network['network_name']) - + vms_by_id[vm['domain']] = dict(id=vm['domain'], macs=dict(), state="shut off") + + vms_by_id[vm['domain']]['network_name'] = network['network_name'] vms_by_id[vm['domain']]['macs'][mac] = True status_json_filename = f"{current_app.config['LIBVIRT_DNSMASQ_PATH']}/{network['virtual_bridge_name']}.status" @@ -205,19 +207,9 @@ class ShellScriptSpoke(VirtualizationInterface): vm_id = vm_id_by_mac[status['mac-address']] vms_by_id[vm_id]['public_ipv4'] = status['ip-address'] else: - current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac") + current_app.logger.warn(f"get_all_by_id: {status['mac-address']} not in vm_id_by_mac") - networks = dict() - for vm in vms_by_id.values(): - if vm['network_name'] not in networks: - networks[vm['network_name']] = [] - - networks[vm['network_name']].append(vm) - - to_return = dict() - to_return[current_app.config['SPOKE_HOST_ID']] = networks - - return to_return + return vms_by_id