From cc349d266fd1dd8af087b3a56139e608e906e760 Mon Sep 17 00:00:00 2001 From: forest Date: Sun, 5 Dec 2021 16:47:37 -0600 Subject: [PATCH] working on support for managing VM state and IP address --- capsulflask/admin.py | 34 ++++++ capsulflask/cli.py | 14 ++- capsulflask/consistency.py | 108 ++++++++++++++++++ capsulflask/console.py | 12 +- capsulflask/db.py | 2 +- capsulflask/db_model.py | 17 +-- capsulflask/hub_model.py | 29 ++--- .../21_down_desired_state.sql | 7 ++ .../schema_migrations/21_up_desired_state.sql | 7 ++ capsulflask/shared.py | 2 +- capsulflask/shell_scripts/list-ids.sh | 3 - capsulflask/shell_scripts/ssh-keyscan.sh | 14 +-- capsulflask/shell_scripts/virsh-list.sh | 13 +++ capsulflask/shell_scripts/virsh-net-list.sh | 3 + capsulflask/spoke_api.py | 6 +- capsulflask/spoke_model.py | 11 +- 16 files changed, 225 insertions(+), 57 deletions(-) create mode 100644 capsulflask/consistency.py create mode 100644 capsulflask/schema_migrations/21_down_desired_state.sql create mode 100644 capsulflask/schema_migrations/21_up_desired_state.sql delete mode 100755 capsulflask/shell_scripts/list-ids.sh create mode 100755 capsulflask/shell_scripts/virsh-list.sh create mode 100644 capsulflask/shell_scripts/virsh-net-list.sh diff --git a/capsulflask/admin.py b/capsulflask/admin.py index 50cc61e..2a80c9a 100644 --- a/capsulflask/admin.py +++ b/capsulflask/admin.py @@ -17,6 +17,10 @@ bp = Blueprint("admin", __name__, url_prefix="/admin") @bp.route("/") @admin_account_required def index(): + + # first create the hosts list w/ ip allocation visualization + # + hosts = get_model().list_hosts_with_networks(None) vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None) network_display_width_px = float(270) @@ -29,6 +33,8 @@ def index(): {'}'} """] + public_ipv4_by_capsul_id = dict() + for kv in hosts.items(): host_id = kv[0] value = kv[1] @@ -45,6 +51,7 @@ def index(): if host_id in vms_by_host_and_network: if network['network_name'] in vms_by_host_and_network[host_id]: for vm in vms_by_host_and_network[host_id][network['network_name']]: + public_ipv4_by_capsul_id[vm['id']] = vm['public_ipv4'] ip_address_int = int(ipaddress.ip_address(vm['public_ipv4'])) if network_start_int <= ip_address_int and ip_address_int <= network_end_int: allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}" @@ -62,10 +69,37 @@ def index(): display_hosts.append(display_host) + + # Now creating the capsuls running status ui + # + + db_vms = get_model().all_vm_ids_with_desired_state() + virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state() + + virt_vms_dict = dict() + for vm in virt_vms: + virt_vms_dict[vm["id"]] = vm["state"] + + in_db_but_not_in_virt = [] + needs_to_be_started = [] + needs_to_be_started_missing_ipv4 = [] + + for vm in db_vms: + if vm["id"] not in virt_vms_dict: + in_db_but_not_in_virt.append(vm["id"]) + elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running": + if vm["id"] in public_ipv4_by_capsul_id: + needs_to_be_started.append({"id": vm["id"], "ipv4": public_ipv4_by_capsul_id[vm["id"]]}) + else: + needs_to_be_started_missing_ipv4.append(vm["id"]) + csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) response_text = render_template( "admin.html", display_hosts=display_hosts, + in_db_but_not_in_virt=in_db_but_not_in_virt, + needs_to_be_started=needs_to_be_started, + needs_to_be_started_missing_ipv4=needs_to_be_started_missing_ipv4, network_display_width_px=network_display_width_px, csp_inline_style_nonce=csp_inline_style_nonce, inline_style='\n'.join(inline_styles) diff --git a/capsulflask/cli.py b/capsulflask/cli.py index 0ab191d..8218961 100644 --- a/capsulflask/cli.py +++ b/capsulflask/cli.py @@ -268,23 +268,25 @@ def notify_users_about_account_balance(): def ensure_vms_and_db_are_synced(): - db_ids = get_model().all_non_deleted_vm_ids() - virt_ids = current_app.config["HUB_MODEL"].list_ids() + db_vms = get_model().all_vm_ids_with_desired_state() + virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state() db_ids_dict = dict() virt_ids_dict = dict() - for id in db_ids: - db_ids_dict[id] = True + for vm in db_vms: + db_ids_dict[vm['id']] = vm['desired_state'] - for id in virt_ids: - virt_ids_dict[id] = True + for vm in virt_vms: + virt_ids_dict[vm['id']] = vm['desired_state'] errors = list() for id in db_ids_dict: if id not in virt_ids_dict: errors.append(f"{id} is in the database but not in the virtualization model") + elif db_ids_dict[id] != virt_ids_dict[id]: + errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model") for id in virt_ids_dict: if id not in db_ids_dict: diff --git a/capsulflask/consistency.py b/capsulflask/consistency.py new file mode 100644 index 0000000..7a86c63 --- /dev/null +++ b/capsulflask/consistency.py @@ -0,0 +1,108 @@ + +from flask import current_app +from capsulflask.db import get_model + +# { +# "capsul-123abc45": { +# "id": "capsul-123abc45", +# "public_ipv4": "123.123.123.123", +# "public_ipv6": "::::", +# "host": "baikal", +# "network_name": "public1", +# "virtual_bridge_name": "virbr1", +# "state": "running" +# }, +# { ... }, +# ... +# } +def get_all_vms_from_db() -> dict: + db_hosts = get_model().list_hosts_with_networks(None) + db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None) + + db_vms_by_id = dict() + + for kv in db_hosts.items(): + host_id = kv[0] + value = kv[1] + for network in value['networks']: + if host_id in db_vms_by_host_and_network and network['network_name'] in db_vms_by_host_and_network[host_id]: + for vm in db_vms_by_host_and_network[host_id][network['network_name']]: + vm['network_name'] = network['network_name'] + vm['virtual_bridge_name'] = network['virtual_bridge_name'] + vm['host'] = host_id + db_vms_by_id[vm['id']] = vm + + # for vm in db_vms: + # if vm["id"] not in db_vms_by_id: + # # TODO + # raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") + # else: + # db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] + + return db_vms_by_id + +def get_all_vms_from_hosts() -> dict: + virt_vms = current_app.config["HUB_MODEL"].virsh_list() + virt_networks = current_app.config["HUB_MODEL"].virsh_netlist() + db_hosts = get_model().list_hosts_with_networks(None) + + virt_vms_by_id = dict() + + for kv in db_hosts.items(): + host_id = kv[0] + value = kv[1] + for network in value['networks']: + + + for vm in db_vms: + if vm["id"] not in db_vms_by_id: + # TODO + raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state") + else: + db_vms_by_id[vm["id"]]["state"] = vm["desired_state"] + + virt_vms = current_app.config["HUB_MODEL"].get_vm_() + +def ensure_vms_and_db_are_synced(): + + + + # Now creating the capsuls running status ui + # + + + + for vm in db_vms: + db_ids_dict[vm['id']] = vm['desired_state'] + + for vm in virt_vms: + virt_ids_dict[vm['id']] = vm['desired_state'] + + errors = list() + + for id in db_ids_dict: + if id not in virt_ids_dict: + errors.append(f"{id} is in the database but not in the virtualization model") + elif db_ids_dict[id] != virt_ids_dict[id]: + errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model") + + for id in virt_ids_dict: + if id not in db_ids_dict: + errors.append(f"{id} is in the virtualization model but not in the database") + + if len(errors) > 0: + email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",") + email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) )) + + current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:") + for error in errors: + current_app.logger.info(f"cron_task: {error}.") + + current_app.config["FLASK_MAIL_INSTANCE"].send( + Message( + "Capsul Consistency Check Failed", + sender=current_app.config["MAIL_DEFAULT_SENDER"], + body="\n".join(errors), + recipients=email_addresses + ) + ) \ No newline at end of file diff --git a/capsulflask/console.py b/capsulflask/console.py index d15155d..f15907a 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -27,12 +27,12 @@ def make_capsul_id(): letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10) return f"capsul-{letters_n_nummers}" -def double_check_capsul_address(id, ipv4, get_ssh_host_keys): +def double_check_capsul_address(id, get_ssh_host_keys): try: result = current_app.config["HUB_MODEL"].get(id, get_ssh_host_keys) - if result != None and result.ipv4 != None and result.ipv4 != ipv4: - ipv4 = result.ipv4 - get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4) + # if result != None and result.ipv4 != None and result.ipv4 != ipv4: + # ipv4 = result.ipv4 + # get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4) if result != None and result.ssh_host_keys != None and get_ssh_host_keys: get_model().update_vm_ssh_host_keys(email=session["account"], id=id, ssh_host_keys=result.ssh_host_keys) @@ -59,7 +59,7 @@ def index(): # for now we are going to check the IP according to the virt model # on every request. this could be done by a background job and cached later on... for vm in vms: - result = double_check_capsul_address(vm["id"], vm["ipv4"], False) + result = double_check_capsul_address(vm["id"], False) if result is not None: vm["ipv4"] = result.ipv4 vm["state"] = result.state @@ -167,7 +167,7 @@ def detail(id): else: needs_ssh_host_keys = "ssh_host_keys" not in vm or len(vm["ssh_host_keys"]) == 0 - vm_from_virt_model = double_check_capsul_address(vm["id"], vm["ipv4"], needs_ssh_host_keys) + vm_from_virt_model = double_check_capsul_address(vm["id"], needs_ssh_host_keys) if vm_from_virt_model is not None: vm["ipv4"] = vm_from_virt_model.ipv4 diff --git a/capsulflask/db.py b/capsulflask/db.py index f216fc6..f6e1d40 100644 --- a/capsulflask/db.py +++ b/capsulflask/db.py @@ -50,7 +50,7 @@ def init_app(app, is_running_server): hasSchemaVersionTable = False actionWasTaken = False schemaVersion = 0 - desiredSchemaVersion = 20 + desiredSchemaVersion = 21 cursor = connection.cursor() diff --git a/capsulflask/db_model.py b/capsulflask/db_model.py index 976985a..1da4f78 100644 --- a/capsulflask/db_model.py +++ b/capsulflask/db_model.py @@ -86,9 +86,9 @@ class DBModel: return hosts - def all_non_deleted_vm_ids(self): - self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL") - return list(map(lambda x: x[0], self.cursor.fetchall())) + def all_vm_ids_with_desired_state(self): + self.cursor.execute("SELECT id, desired_state FROM vms WHERE deleted IS NULL") + return list(map(lambda x: {"id": x[0], "desired_state": x[1]}, self.cursor.fetchall())) def operating_systems_dict(self): self.cursor.execute("SELECT id, template_image_file_name, description FROM os_images WHERE deprecated = FALSE") @@ -332,7 +332,7 @@ class DBModel: def list_hosts_with_networks(self, host_id: str): query = """ - SELECT hosts.id, hosts.last_health_check, host_network.network_name, + SELECT hosts.id, hosts.last_health_check, host_network.network_name, host_network.virtual_bridge_name, host_network.public_ipv4_cidr_block, host_network.public_ipv4_first_usable_ip, host_network.public_ipv4_last_usable_ip FROM hosts JOIN host_network ON host_network.host = hosts.id @@ -354,10 +354,11 @@ class DBModel: hosts[row[0]] = dict(last_health_check=row[1], networks=[]) hosts[row[0]]["networks"].append(dict( - network_name=row[2], - public_ipv4_cidr_block=row[3], - public_ipv4_first_usable_ip=row[4], - public_ipv4_last_usable_ip=row[5] + network_name=row[2], + virtual_bridge_name=row[3], + public_ipv4_cidr_block=row[4], + public_ipv4_first_usable_ip=row[5], + public_ipv4_last_usable_ip=row[6] )) return hosts diff --git a/capsulflask/hub_model.py b/capsulflask/hub_model.py index 73f60f5..4431a7f 100644 --- a/capsulflask/hub_model.py +++ b/capsulflask/hub_model.py @@ -37,8 +37,8 @@ class MockHub(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4) - def list_ids(self) -> list: - return get_model().all_non_deleted_vm_ids() + def get_all_by_host_and_network(self) -> dict: + return get_model().non_deleted_vms_by_host_and_network() def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) @@ -164,29 +164,24 @@ class CapsulFlaskHub(VirtualizationInterface): return None - def list_ids(self) -> list: + def get_all_by_host_and_network(self) -> dict: online_hosts = get_model().get_online_hosts() - payload = json.dumps(dict(type="list_ids")) + payload = json.dumps(dict(type="get_all_by_host_and_network")) results = self.synchronous_operation(online_hosts, None, payload) - to_return = [] + to_return = dict() for i in range(len(results)): host = online_hosts[i] result = results[i] try: result_body = json.loads(result.body) - if isinstance(result_body, dict) and 'ids' in result_body and isinstance(result_body['ids'], list): - all_valid = True - for id in result_body['ids']: - try: - validate_capsul_id(id) - to_return.append(id) - except: - all_valid = False - if not all_valid: - current_app.logger.error(f"""error reading ids for list_ids operation, host {host.id}""") + + has_host = isinstance(result_body, dict) and host.id in result_body and isinstance(result_body[host.id], dict) + has_networks = has_host and 'networks' in result_body[host.id] and isinstance(result_body[host.id]['networks'], dict) + if has_host and has_networks: + to_return[host.id] = result_body[host.id]['networks'] else: - result_json_string = json.dumps({"error_message": "invalid response, missing 'ids' list"}) - current_app.logger.error(f"""missing 'ids' list for list_ids operation, host {host.id}""") + # result_json_string = json.dumps({"error_message": "invalid response, missing 'networks' list"}) + current_app.logger.error(f"""missing 'networks' list for get_all_by_host_and_network operation, host {host.id}""") except: # no need to do anything here since if it cant be parsed then generic_operation will handle it. pass diff --git a/capsulflask/schema_migrations/21_down_desired_state.sql b/capsulflask/schema_migrations/21_down_desired_state.sql new file mode 100644 index 0000000..9bba8da --- /dev/null +++ b/capsulflask/schema_migrations/21_down_desired_state.sql @@ -0,0 +1,7 @@ + + + +ALTER TABLE vms DROP COLUMN desired_state; + +UPDATE schemaversion SET version = 20; + diff --git a/capsulflask/schema_migrations/21_up_desired_state.sql b/capsulflask/schema_migrations/21_up_desired_state.sql new file mode 100644 index 0000000..357db8c --- /dev/null +++ b/capsulflask/schema_migrations/21_up_desired_state.sql @@ -0,0 +1,7 @@ + + + +ALTER TABLE vms ADD COLUMN desired_state TEXT DEFAULT 'running'; + +UPDATE schemaversion SET version = 21; + diff --git a/capsulflask/shared.py b/capsulflask/shared.py index ba72900..5285275 100644 --- a/capsulflask/shared.py +++ b/capsulflask/shared.py @@ -31,7 +31,7 @@ class VirtualizationInterface: def get(self, id: str) -> VirtualMachine: pass - def list_ids(self) -> list: + def get_all_by_host_and_network(self) -> dict: pass def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list): diff --git a/capsulflask/shell_scripts/list-ids.sh b/capsulflask/shell_scripts/list-ids.sh deleted file mode 100755 index 1f1c3b5..0000000 --- a/capsulflask/shell_scripts/list-ids.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -virsh list --all | grep running | grep -v ' Id' | grep -v -- '----' | awk '{print $2}' | sort \ No newline at end of file diff --git a/capsulflask/shell_scripts/ssh-keyscan.sh b/capsulflask/shell_scripts/ssh-keyscan.sh index 909ac40..6975d51 100755 --- a/capsulflask/shell_scripts/ssh-keyscan.sh +++ b/capsulflask/shell_scripts/ssh-keyscan.sh @@ -8,15 +8,15 @@ if echo "$ip_address" | grep -vqE '^([0-9]{1,3}\.){3}[0-9]{1,3}$'; then fi printf '[' -DELIMITER="" +delimiter="" ssh-keyscan "$ip_address" 2>/dev/null | while read -r line; do if echo "$line" | grep -qE "^$ip_address"' +(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$'; then - KEY_CONTENT="$(echo "$line" | awk '{ print $2 " " $3 }')" - FINGERPRINT_OUTPUT="$(echo "$KEY_CONTENT" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')" - SHA256_HASH="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $1 }')" - KEY_TYPE="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $2 }')" - printf '%s\n {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$DELIMITER" "$KEY_TYPE" "$KEY_CONTENT" "$SHA256_HASH" - DELIMITER="," + key_content="$(echo "$line" | awk '{ print $2 " " $3 }')" + fingerprint_output="$(echo "$key_content" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')" + sha256_hash="$(echo "$fingerprint_output" | awk '{ print $1 }')" + key_type="$(echo "$fingerprint_output" | awk '{ print $2 }')" + printf '%s\n {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$delimiter" "$key_type" "$key_content" "$sha256_hash" + delimiter="," fi done printf '\n]\n' diff --git a/capsulflask/shell_scripts/virsh-list.sh b/capsulflask/shell_scripts/virsh-list.sh new file mode 100755 index 0000000..11147c1 --- /dev/null +++ b/capsulflask/shell_scripts/virsh-list.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +printf '[' +delimiter="" +virsh list --all | while read -r line; do + if echo "$line" | grep -qE '(running)|(shut off)'; then + capsul_id="$(echo "$line" | awk '{ print $2 }')" + capsul_state="$(echo "$line" | sed -E 's/.*((running)|(shut off))\w*/\1/')" + printf '%s\n {"id":"%s", "state":"%s"}' "$delimiter" "$capsul_id" "$capsul_state" + delimiter="," + fi +done +printf '\n]\n' diff --git a/capsulflask/shell_scripts/virsh-net-list.sh b/capsulflask/shell_scripts/virsh-net-list.sh new file mode 100644 index 0000000..65b47c8 --- /dev/null +++ b/capsulflask/shell_scripts/virsh-net-list.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +virsh net-list --all | tail -n +3 | awk '{ print $1 }' \ No newline at end of file diff --git a/capsulflask/spoke_api.py b/capsulflask/spoke_api.py index c178572..54af3a2 100644 --- a/capsulflask/spoke_api.py +++ b/capsulflask/spoke_api.py @@ -49,7 +49,7 @@ def operation_impl(operation_id: int): handlers = { "capacity_avaliable": handle_capacity_avaliable, "get": handle_get, - "list_ids": handle_list_ids, + "get_all_by_host_and_network": handle_get_all_by_host_and_network, "create": handle_create, "destroy": handle_destroy, "vm_state_command": handle_vm_state_command, @@ -96,8 +96,8 @@ def handle_get(operation_id, request_body): return jsonify(dict(assignment_status="assigned", id=vm.id, host=vm.host, state=vm.state, ipv4=vm.ipv4, ipv6=vm.ipv6, ssh_host_keys=vm.ssh_host_keys)) -def handle_list_ids(operation_id, request_body): - return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].list_ids())) +def handle_get_all_by_host_and_network(operation_id, request_body): + return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].get_all_by_host_and_network())) def handle_create(operation_id, request_body): if not operation_id: diff --git a/capsulflask/spoke_model.py b/capsulflask/spoke_model.py index 9ab0d3a..e38aef9 100644 --- a/capsulflask/spoke_model.py +++ b/capsulflask/spoke_model.py @@ -38,8 +38,8 @@ class MockSpoke(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running") - def list_ids(self) -> list: - return get_model().all_non_deleted_vm_ids() + def get_all_by_host_and_network(self) -> dict: + return get_model().non_deleted_vms_by_host_and_network(None) def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str): validate_capsul_id(id) @@ -133,10 +133,11 @@ class ShellScriptSpoke(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr) - def list_ids(self) -> list: - completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True) + def get_all_by_host_and_network(self) -> list: + # TODO implement this + completedProcess = run([join(current_app.root_path, 'shell_scripts/virsh-list.sh')], capture_output=True) self.validate_completed_process(completedProcess) - return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() )) + return json.loads(completedProcess.stdout.decode("utf-8")) def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str): validate_capsul_id(id)