trying to get admin tools in place

capsulflask/admin.py

@@ -11,6 +11,7 @@ from nanoid import generate
 from capsulflask.metrics import durations as metric_durations
 from capsulflask.auth import admin_account_required
 from capsulflask.db import get_model
+from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
 from capsulflask.shared import my_exec_info_message
 
 bp = Blueprint("admin", __name__, url_prefix="/admin")
@@ -48,12 +49,13 @@ def index():
     else:
       return abort(400, "unknown form action")
 
+  # moving on from the form post stuff... 
 
   # first create the hosts list w/ ip allocation visualization from the database
   # 
    
   db_hosts = get_model().list_hosts_with_networks(None)
-  db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
+  db_vms_by_id = get_all_vms_from_db()
   network_display_width_px = float(270)
   #operations = get_model().list_all_operations()
 
@@ -64,7 +66,13 @@ def index():
   {'}'}
   """]
 
-  db_vm_by_id = dict()
+  db_vms_by_host_network = dict()
+  for vm in db_vms_by_id.values():
+    host_network_key = f"{vm['host']}_{vm['network_name']}" 
+    if host_network_key not in db_vms_by_host_network:
+      db_vms_by_host_network[host_network_key] = []
+    db_vms_by_host_network[host_network_key].append(vm)
+
 
   for kv in db_hosts.items():
     host_id = kv[0]
@@ -79,13 +87,9 @@ def index():
       network['allocations'] = []
       network_addresses_width = float((network_end_int-network_start_int)+1)
 
-      if host_id in db_vms_by_host_and_network:
+      host_network_key = f"{host_id}_{network['network_name']}" 
-        if network['network_name'] in db_vms_by_host_and_network[host_id]:
+      if host_network_key in db_vms_by_host_network:
-          for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
+        for vm in db_vms_by_host_network[host_network_key]:
-            vm['network_name'] = network['network_name']
-            vm['virtual_bridge_name'] = network['virtual_bridge_name']
-            vm['host'] = host_id
-            db_vm_by_id[vm['id']] = vm
           ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
           if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
             allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
@@ -104,28 +108,42 @@ def index():
     display_hosts.append(display_host)
 
 
-  # Now creating the capsuls running status ui
+  # Now creating the capsul consistency / running status ui
   # 
 
-  virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
+  virt_vms_by_id = get_all_vms_from_hosts()
   
-  # virt_vms_dict = dict()
+  virt_vm_id_by_ipv4 = dict()
-  # for vm in virt_vms:
+  for vm_id, virt_vm in virt_vms_by_id.items():
-  #   virt_vms_dict[vm["id"]] = vm["state"]
+    virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
 
-  # in_db_but_not_in_virt = []
+  db_vm_id_by_ipv4 = dict()
-  # needs_to_be_started = []
+  for vm_id, db_vm in db_vms_by_id.items():
-  # needs_to_be_started_missing_ipv4 = []
+    db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
+
+  in_db_but_not_in_virt = []
+  state_not_equal_to_desired_state = []
+  stole_someone_elses_ip_and_own_ip_avaliable = []
+  stole_someone_elses_ip_but_own_ip_also_stolen = []
+  has_wrong_ip = []
+
+  for vm_id, db_vm in db_vms_by_id.items():
+    if vm_id not in virt_vms_by_id:
+      in_db_but_not_in_virt.append(db_vm)
+    elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
+      db_vm["state"] = virt_vms_by_id[vm_id]['state']
+      state_not_equal_to_desired_state.append(db_vm)
+    elif virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
+      db_vm["desired_ipv4"] = db_vm["public_ipv4"]
+      db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
+      if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
+        if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
+          stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
+        else:
+          stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
+      
+      has_wrong_ip.append(db_vm)
 
-  # for vm in db_vms:
-  #   if vm["id"] not in virt_vms_dict:
-  #     in_db_but_not_in_virt.append(vm["id"])
-  #   elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
-  #     if vm["id"] in db_vm_by_id:
-  #       needs_to_be_started.append(db_vm_by_id[vm["id"]])
-  #     else:
-  #       needs_to_be_started_missing_ipv4.append(vm["id"])
-  #   elif vm["ipv4"] != current_ipv4
 
   # current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
 
@@ -142,8 +160,11 @@ def index():
     csp_inline_style_nonce=csp_inline_style_nonce,
     inline_style='\n'.join(inline_styles),
 
-    db_vms_by_host_and_network=json.dumps(db_vms_by_host_and_network),
+    in_db_but_not_in_virt=in_db_but_not_in_virt,
-    virt_vms_by_host_and_network=json.dumps(virt_vms_by_host_and_network),
+    state_not_equal_to_desired_state=state_not_equal_to_desired_state,
+    stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
+    stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
+    has_wrong_ip=has_wrong_ip
   )
 
   response = make_response(response_text)

capsulflask/consistency.py

@@ -10,7 +10,7 @@ from capsulflask.db import get_model
 #     "host": "baikal",
 #     "network_name": "public1",
 #     "virtual_bridge_name": "virbr1",
-#     "state": "running"
+#     "desired_state": "running"
 #   },
 #   { ... },
 #   ...
@@ -41,8 +41,10 @@ def get_all_vms_from_db() -> dict:
 
   return db_vms_by_id
 
+
+# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
 def get_all_vms_from_hosts() -> dict:
-  virt_vms = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
+  virt_vms_by_host_and_network = current_app.config["HUB_MODEL"].get_all_by_host_and_network()
   #virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
   db_hosts = get_model().list_hosts_with_networks(None)
   
@@ -52,57 +54,55 @@ def get_all_vms_from_hosts() -> dict:
     host_id = kv[0]
     value = kv[1]
     for network in value['networks']:
+      if host_id in virt_vms_by_host_and_network and network['network_name'] in virt_vms_by_host_and_network[host_id]:
+        for vm in virt_vms_by_host_and_network[host_id][network['network_name']]:
+          vm['network_name'] = network['network_name']
+          vm['virtual_bridge_name'] = network['virtual_bridge_name']
+          vm['host'] = host_id
+          virt_vms_by_id[vm['id']] = vm
 
-
+  return virt_vms_by_id
-  for vm in db_vms:
-    if vm["id"] not in db_vms_by_id:
-      # TODO 
-      raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") 
-    else:
-      db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] 
-
-  virt_vms = current_app.config["HUB_MODEL"].get_vm_()
 
 def ensure_vms_and_db_are_synced():
   
   
-
+  pass
-  # Now creating the capsuls running status ui
+  # # Now creating the capsuls running status ui
-  # 
+  # # 
 
 
 
-  for vm in db_vms:
+  # for vm in db_vms:
-    db_ids_dict[vm['id']] = vm['desired_state']
+  #   db_ids_dict[vm['id']] = vm['desired_state']
   
-  for vm in virt_vms:
+  # for vm in virt_vms:
-    virt_ids_dict[vm['id']] = vm['desired_state']
+  #   virt_ids_dict[vm['id']] = vm['desired_state']
 
-  errors = list()
+  # errors = list()
 
-  for id in db_ids_dict:
+  # for id in db_ids_dict:
-    if id not in virt_ids_dict:
+  #   if id not in virt_ids_dict:
-      errors.append(f"{id} is in the database but not in the virtualization model")
+  #     errors.append(f"{id} is in the database but not in the virtualization model")
-    elif db_ids_dict[id] != virt_ids_dict[id]:
+  #   elif db_ids_dict[id] != virt_ids_dict[id]:
-      errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
+  #     errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
 
-  for id in virt_ids_dict:
+  # for id in virt_ids_dict:
-    if id not in db_ids_dict:
+  #   if id not in db_ids_dict:
-      errors.append(f"{id} is in the virtualization model but not in the database")
+  #     errors.append(f"{id} is in the virtualization model but not in the database")
 
-  if len(errors) > 0:
+  # if len(errors) > 0:
-    email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
+  #   email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
-    email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
+  #   email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
 
-    current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
+  #   current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
-    for error in errors:
+  #   for error in errors:
-      current_app.logger.info(f"cron_task:      {error}.")
+  #     current_app.logger.info(f"cron_task:      {error}.")
       
-    current_app.config["FLASK_MAIL_INSTANCE"].send(
+  #   current_app.config["FLASK_MAIL_INSTANCE"].send(
-        Message(
+  #       Message(
-          "Capsul Consistency Check Failed",
+  #         "Capsul Consistency Check Failed",
-          sender=current_app.config["MAIL_DEFAULT_SENDER"],
+  #         sender=current_app.config["MAIL_DEFAULT_SENDER"],
-          body="\n".join(errors),
+  #         body="\n".join(errors),
-          recipients=email_addresses
+  #         recipients=email_addresses
-        )
+  #       )
-    )
+  #   )

capsulflask/shell_scripts/virsh-net-list.sh

@@ -7,7 +7,7 @@ virsh net-list --all | tail -n +3 | awk '{ print $1 }' | while read -r network_n
   if [ "$network_name" != "" ]; then
     virtual_bridge_name="$(virsh net-info "$network_name" | grep -E  '^Bridge:' | awk '{ print $2 }')"
     capsul_state="$(echo "$line" | sed -E 's/^ *[0-9-]+ +[^ ]+ +//')"
-    printf '%s\n  {"name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
+    printf '%s\n  {"network_name":"%s", "virtual_bridge_name":"%s"}' "$delimiter" "$network_name" "$virtual_bridge_name"
     delimiter=","
   fi
 done

capsulflask/spoke_model.py

@@ -184,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
               else:
                 current_app.logger.warn(f"get_all_by_host_and_network: '{vm['domain']}' not in vm_state_by_id, defaulting to 'shut off'")
 
-              vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network=network['name'])
+              vms_by_id[vm['domain']] = dict(macs=dict(), state=vm_state, network_name=network['network_name'])
                
             vms_by_id[vm['domain']]['macs'][mac] = True
       
@@ -199,14 +199,12 @@ class ShellScriptSpoke(VirtualizationInterface):
         for status in statuses:
           if status['mac-address'] in vm_id_by_mac:
             vm_id = vm_id_by_mac[status['mac-address']]
-            vms_by_id[vm_id]['ipv4'] = status['ip-address']
+            vms_by_id[vm_id]['public_ipv4'] = status['ip-address']
           else:
             current_app.logger.warn(f"get_all_by_host_and_network: {status['mac-address']} not in vm_id_by_mac")
 
     networks = dict()
-    for vm_id in vms_by_id:
+    for vm in vms_by_id.values():
-      vm = vms_by_id[vm_id]
-      
       if vm['network'] not in networks:
         networks[vm['network']] = []
 

capsulflask/templates/admin.html

@@ -36,28 +36,97 @@
     
     <hr/>
   {% endfor %}
+</div>
 
+{% if in_db_but_not_in_virt|length > 0 %}
+<div class="third-margin">
+  <h1>🚨 in the database but not in the virtualization model 🚨</h1>
+  {% for vm in in_db_but_not_in_virt %}
   <div class="row">
-    <h1>db_vms_by_host_and_network</h1>
+    {{vm['id']}} {{vm['public_ipv4']}}
   </div>
+  {% endfor %}
+  <hr/>
+</div>
+{% endif %}
+
+
+
+{% if state_not_equal_to_desired_state|length > 0 %}
+<div class="third-margin">
+  <h1>😴 vm state != desired state 😴</h1>
+  {% for vm in state_not_equal_to_desired_state %}
   <div class="row">
-    <pre>
+    <div>{{vm['id']}}: state={{vm['state']}} desired_state={{vm['desired_state']}}</div>
-{{db_vms_by_host_and_network}}
+    <form method="post">
-    </pre>
+      <input type="hidden" name="action" value="set_state"></input> 
+      <input type="hidden" name="id" value="{{vm['id']}}"></input> 
+      <input type="hidden" name="state" value="{{vm['desired_state']}}"></input> 
+      <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+      <input type="submit" value="🚦 START/STOP"/>
+    </form>
   </div>
+  {% endfor %}
+  <hr/>
+</div>
+{% endif %}
 
+
+
+{% if stole_someone_elses_ip_and_own_ip_avaliable|length > 0 %}
+<div class="third-margin">
+  <h1>👻 stole someone elses ip and own desired ip is avaliable 👻</h1>
+  {% for vm in stole_someone_elses_ip_and_own_ip_avaliable %}
   <div class="row">
-    <h1>virt_vms_by_host_and_network</h1>
+    <div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
+    <form method="post">
+      <input type="hidden" name="action" value="dhcp_reset"></input> 
+      <input type="hidden" name="id" value="{{vm['id']}}"></input> 
+      <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+      <input type="submit" value="🔨 DHCP RESET"/>
+    </form>
   </div>
+  {% endfor %}
+  <hr/>
+</div>
+{% endif %}
+
+{% if has_wrong_ip|length > 0 %}
+<div class="third-margin">
+  <h1>🥴 has wrong ip address 🥴</h1>
+  {% for vm in has_wrong_ip %}
   <div class="row">
-    <pre>
+    <div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
-{{virt_vms_by_host_and_network}}
+    <form method="post">
-    </pre>
+      <input type="hidden" name="action" value="dhcp_reset"></input> 
+      <input type="hidden" name="id" value="{{vm['id']}}"></input> 
+      <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+      <input type="submit" value="🔨 DHCP RESET"/>
+    </form>
   </div>
-
+  {% endfor %}
-
+  <hr/>
-  
 </div>
+{% endif %}
+
+
+{% if stole_someone_elses_ip_but_own_ip_also_stolen|length > 0 %}
+<div class="third-margin">
+  <h1>💀 stole someone elses ip but own desired ip was also stolen 💀</h1>
+  {% for vm in stole_someone_elses_ip_but_own_ip_also_stolen %}
+  <div class="row">
+    <div>{{vm['id']}}: current_ipv4={{vm['current_ipv4']}} desired_ipv4={{vm['desired_ipv4']}}</div>
+    <form method="post">
+      <input type="hidden" name="action" value="stop_and_expire"></input> 
+      <input type="hidden" name="id" value="{{vm['id']}}"></input> 
+      <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+      <input type="submit" value="🛑 STOP AND EXPIRE DHCP LEASE"/>
+    </form>
+  </div>
+  {% endfor %}
+  <hr/>
+</div>
+{% endif %}
 
 <div class="third-margin">
   <div class="row">
@@ -74,6 +143,7 @@
   </div>
  
 </div>
+
 {% endblock %}
 
 {% block pagesource %}/templates/admin.html{% endblock %}