refactor get_all_by_host_and_network to get_all_by_id

capsulflask/consistency.py

@@ -43,23 +43,26 @@ def get_all_vms_from_db() -> dict:
 
 
 # this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
-def get_all_vms_from_hosts() -> dict:
+def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
-  virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
+  virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id()
-  #virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
   db_hosts = get_model().list_hosts_with_networks(None)
 
-  virt_vms_by_id = dict()
+  for vm_id, virt_vm in virt_vms_by_id.items():
+    if vm_id in db_vms_by_id:
+      if 'host' not in virt_vm:
+        virt_vm['host'] = db_vms_by_id[vm_id]['host']
+      if 'network_name' not in virt_vm:
+        virt_vm['network_name'] = db_vms_by_id[vm_id]['network_name']
+    else:
+      if 'host' not in virt_vm:
+        virt_vm['host'] = "unknown"
+      if 'network_name' not in virt_vm:
+        virt_vm['network_name'] = "unknown"
 
-  for kv in db_hosts.items():
+    if virt_vm['host'] in db_hosts and virt_vm['network_name'] in db_hosts[virt_vm['host']]:
-    host_id = kv[0]
+      virt_vm['virtual_bridge_name'] = db_hosts[virt_vm['host']][virt_vm['network_name']]['virtual_bridge_name']
-    value = kv[1]
+    else:
-    for network in value['networks']:
+      virt_vm['virtual_bridge_name'] = "unknown"
-      if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
-        for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
-          vm['network_name'] = network['network_name']
-          vm['virtual_bridge_name'] = network['virtual_bridge_name']
-          vm['host'] = host_id
-          virt_vms_by_id[vm['id']] = vm
 
   return virt_vms_by_id
 

capsulflask/db_model.py

@@ -81,7 +81,7 @@ class DBModel:
         hosts[host_id][network_name] = []
       
       hosts[host_id][network_name].append(
-        dict(id=row[0], email=row[1], desired_state=row[2], public_ipv4=row[5], public_ipv6=row[6])
+        dict(id=row[0], email=row[1], host=host_id, network_name=network_name, desired_state=row[2], public_ipv4=row[5], public_ipv6=row[6])
       )
 
     return hosts

capsulflask/hub_model.py

@@ -37,8 +37,15 @@ class MockHub(VirtualizationInterface):
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4)
 
-  def get_all_by_host_and_network(self) -> dict:
+  def get_all_by_id(self) -> dict:
-    return get_model().non_deleted_vms_by_host_and_network()
+    by_host_and_network = get_model().non_deleted_vms_by_host_and_network()
+    to_return = dict()
+    for host in by_host_and_network.values():
+      for network in host.values():
+        for vm in network:
+          to_return[vm['id']] = vm
+
+    return to_return
 
   def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
     validate_capsul_id(id)
@@ -167,9 +174,9 @@ class CapsulFlaskHub(VirtualizationInterface):
 
     return None
 
-  def get_all_by_host_and_network(self) -> dict:
+  def get_all_by_id(self) -> dict:
     online_hosts = get_model().get_online_hosts()
-    payload = json.dumps(dict(type="get_all_by_host_and_network"))
+    payload = json.dumps(dict(type="get_all_by_id"))
     results = self.synchronous_operation(online_hosts, None, payload)
     to_return = dict()
     for i in range(len(results)):
@@ -177,14 +184,9 @@ class CapsulFlaskHub(VirtualizationInterface):
       result = results[i]
       try:
         result_body = json.loads(result.body)
-
+        for id, vm in result_body.items():
-        has_hosts = isinstance(result_body, dict) and "hosts" in result_body and isinstance(result_body["hosts"], dict)
+          vm['host'] = host.id
-        has_current_host = has_hosts and isinstance(result_body["hosts"], dict) and host.id in result_body["hosts"] and isinstance(result_body["hosts"][host.id], dict)
+          to_return[id] = vm
-        if has_current_host:
-          to_return[host.id] = result_body["hosts"][host.id]
-        else:
-          # result_json_string = json.dumps({"error_message": "invalid response, missing 'networks' list"})
-          current_app.logger.error(f"""missing 'hosts' dict for get_all_by_host_and_network operation, host {host.id}""")
       except:
         # no need to do anything here since if it cant be parsed then generic_operation will handle it.
         pass

capsulflask/shared.py

@@ -32,7 +32,7 @@ class VirtualizationInterface:
   def get(self, id: str) -> VirtualMachine:
     pass
 
-  def get_all_by_host_and_network(self) -> dict:
+  def get_all_by_id(self) -> dict:
     pass
 
   def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list):

capsulflask/spoke_api.py

@@ -49,7 +49,7 @@ def operation_impl(operation_id: int):
     handlers = {
       "capacity_avaliable": handle_capacity_avaliable,
       "get": handle_get,
-      "get_all_by_host_and_network": handle_get_all_by_host_and_network,
+      "get_all_by_id": handle_get_all_by_id,
       "create": handle_create,
       "destroy": handle_destroy,
       "vm_state_command": handle_vm_state_command,
@@ -97,8 +97,8 @@ def handle_get(operation_id, request_body):
   
   return jsonify(dict(assignment_status="assigned", id=vm.id, host=vm.host, state=vm.state, ipv4=vm.ipv4, ipv6=vm.ipv6, ssh_host_keys=vm.ssh_host_keys))
 
-def handle_get_all_by_host_and_network(operation_id, request_body) -> dict:
+def handle_get_all_by_id(operation_id, request_body) -> dict:
-  return jsonify(dict(assignment_status="assigned", hosts=current_app.config['SPOKE_MODEL'].get_all_by_host_and_network()))
+  return jsonify(dict(assignment_status="assigned", vms=current_app.config['SPOKE_MODEL'].get_all_by_id()))
 
 def handle_create(operation_id, request_body):
   if not operation_id:

capsulflask/spoke_model.py

@@ -39,9 +39,15 @@ class MockSpoke(VirtualizationInterface):
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
 
-  def get_all_by_host_and_network(self) -> dict:
+  def get_all_by_id(self) -> dict:
-    to_return = get_model().non_deleted_vms_by_host_and_network(None)
+    by_host_and_network = get_model().non_deleted_vms_by_host_and_network()
-    current_app.logger.info(f"MOCK get_all_by_host_and_network: {json.dumps(to_return)}")
+    to_return = dict()
+    for host in by_host_and_network.values():
+      for network in host.values():
+        for vm in network:
+          to_return[vm['id']] = vm
+
+    current_app.logger.info(f"MOCK get_all_by_id: {json.dumps(to_return)}")
     return to_return
 
   def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
@@ -140,7 +146,7 @@ class ShellScriptSpoke(VirtualizationInterface):
 
     return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr)
 
-  def get_all_by_host_and_network(self) -> dict:
+  def get_all_by_id(self) -> dict:
 
     vm_list_process = run([join(current_app.root_path, 'shell_scripts/virsh-list.sh')], capture_output=True)
     self.validate_completed_process(vm_list_process)
@@ -150,9 +156,9 @@ class ShellScriptSpoke(VirtualizationInterface):
 
     #current_app.logger.info(f"list_of_vms: {json.dumps(list_of_vms)}")
 
-    vm_state_by_id = dict()
+    vms_by_id = dict()
     for vm in list_of_vms:
-      vm_state_by_id[vm['id']] = vm['state']
+      vms_by_id[vm['id']] = dict(id=vm['id'], macs=dict(), state=vm['state'])
 
     net_list_process = run([join(current_app.root_path, 'shell_scripts/virsh-net-list.sh')], capture_output=True)
     self.validate_completed_process(net_list_process)
@@ -162,7 +168,6 @@ class ShellScriptSpoke(VirtualizationInterface):
 
     #current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
 
-    vms_by_id = dict()
     vm_id_by_mac = dict()
     for network in list_of_networks:
 
@@ -182,14 +187,11 @@ class ShellScriptSpoke(VirtualizationInterface):
               raise Exception(f"the mac address '{mac}' is used by both '{vm_id_by_mac[mac]}' and '{vm['domain']}'")
 
             if vm['domain'] not in vms_by_id:
-              vm_state = 'shut off'
+              current_app.logger.warn(f"'{vm['domain']}' was in dnsmask but not in libvirt, defaulting to 'shut off' state")
-              if vm['domain'] in vm_state_by_id:
-                vm_state = vm_state_by_id[vm['domain']]
-              else:
-                current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
 
-              vms_by_id[vm['domain']] = dict(id=vm['domain'], macs=dict(), state=vm_state, network_name=network['network_name'])
+              vms_by_id[vm['domain']] = dict(id=vm['domain'], macs=dict(), state="shut off")
             
+            vms_by_id[vm['domain']]['network_name'] = network['network_name']
             vms_by_id[vm['domain']]['macs'][mac] = True
       
       status_json_filename = f"{current_app.config['LIBVIRT_DNSMASQ_PATH']}/{network['virtual_bridge_name']}.status"
@@ -205,19 +207,9 @@ class ShellScriptSpoke(VirtualizationInterface):
             vm_id = vm_id_by_mac[status['mac-address']]
             vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
           else:
-            current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
+            current_app.logger.warn(f"get_all_by_id: {status['mac-address']} not in vm_id_by_mac")
 
-    networks = dict()
+    return vms_by_id
-    for vm in vms_by_id.values():
-      if vm['network_name'] not in networks:
-        networks[vm['network_name']] = []
-
-      networks[vm['network_name']].append(vm)
-
-    to_return = dict()
-    to_return[current_app.config['SPOKE_HOST_ID']] = networks
-
-    return to_return