working on support for managing VM state and IP address

This commit is contained in:
forest 2021-12-05 16:47:37 -06:00
parent 9d74c99ce8
commit cc349d266f
16 changed files with 225 additions and 57 deletions

View File

@ -17,6 +17,10 @@ bp = Blueprint("admin", __name__, url_prefix="/admin")
@bp.route("/")
@admin_account_required
def index():
# first create the hosts list w/ ip allocation visualization
#
hosts = get_model().list_hosts_with_networks(None)
vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
network_display_width_px = float(270)
@ -29,6 +33,8 @@ def index():
{'}'}
"""]
public_ipv4_by_capsul_id = dict()
for kv in hosts.items():
host_id = kv[0]
value = kv[1]
@ -45,6 +51,7 @@ def index():
if host_id in vms_by_host_and_network:
if network['network_name'] in vms_by_host_and_network[host_id]:
for vm in vms_by_host_and_network[host_id][network['network_name']]:
public_ipv4_by_capsul_id[vm['id']] = vm['public_ipv4']
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
@ -62,10 +69,37 @@ def index():
display_hosts.append(display_host)
# Now creating the capsuls running status ui
#
db_vms = get_model().all_vm_ids_with_desired_state()
virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state()
virt_vms_dict = dict()
for vm in virt_vms:
virt_vms_dict[vm["id"]] = vm["state"]
in_db_but_not_in_virt = []
needs_to_be_started = []
needs_to_be_started_missing_ipv4 = []
for vm in db_vms:
if vm["id"] not in virt_vms_dict:
in_db_but_not_in_virt.append(vm["id"])
elif vm["desired_state"] == "running" and virt_vms_dict[vm["id"]] != "running":
if vm["id"] in public_ipv4_by_capsul_id:
needs_to_be_started.append({"id": vm["id"], "ipv4": public_ipv4_by_capsul_id[vm["id"]]})
else:
needs_to_be_started_missing_ipv4.append(vm["id"])
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
response_text = render_template(
"admin.html",
display_hosts=display_hosts,
in_db_but_not_in_virt=in_db_but_not_in_virt,
needs_to_be_started=needs_to_be_started,
needs_to_be_started_missing_ipv4=needs_to_be_started_missing_ipv4,
network_display_width_px=network_display_width_px,
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles)

View File

@ -268,23 +268,25 @@ def notify_users_about_account_balance():
def ensure_vms_and_db_are_synced():
db_ids = get_model().all_non_deleted_vm_ids()
virt_ids = current_app.config["HUB_MODEL"].list_ids()
db_vms = get_model().all_vm_ids_with_desired_state()
virt_vms = current_app.config["HUB_MODEL"].list_ids_with_desired_state()
db_ids_dict = dict()
virt_ids_dict = dict()
for id in db_ids:
db_ids_dict[id] = True
for vm in db_vms:
db_ids_dict[vm['id']] = vm['desired_state']
for id in virt_ids:
virt_ids_dict[id] = True
for vm in virt_vms:
virt_ids_dict[vm['id']] = vm['desired_state']
errors = list()
for id in db_ids_dict:
if id not in virt_ids_dict:
errors.append(f"{id} is in the database but not in the virtualization model")
elif db_ids_dict[id] != virt_ids_dict[id]:
errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
for id in virt_ids_dict:
if id not in db_ids_dict:

108
capsulflask/consistency.py Normal file
View File

@ -0,0 +1,108 @@
from flask import current_app
from capsulflask.db import get_model
# {
# "capsul-123abc45": {
# "id": "capsul-123abc45",
# "public_ipv4": "123.123.123.123",
# "public_ipv6": "::::",
# "host": "baikal",
# "network_name": "public1",
# "virtual_bridge_name": "virbr1",
# "state": "running"
# },
# { ... },
# ...
# }
def get_all_vms_from_db() -> dict:
db_hosts = get_model().list_hosts_with_networks(None)
db_vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
db_vms_by_id = dict()
for kv in db_hosts.items():
host_id = kv[0]
value = kv[1]
for network in value['networks']:
if host_id in db_vms_by_host_and_network and network['network_name'] in db_vms_by_host_and_network[host_id]:
for vm in db_vms_by_host_and_network[host_id][network['network_name']]:
vm['network_name'] = network['network_name']
vm['virtual_bridge_name'] = network['virtual_bridge_name']
vm['host'] = host_id
db_vms_by_id[vm['id']] = vm
# for vm in db_vms:
# if vm["id"] not in db_vms_by_id:
# # TODO
# raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
# else:
# db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
return db_vms_by_id
def get_all_vms_from_hosts() -> dict:
virt_vms = current_app.config["HUB_MODEL"].virsh_list()
virt_networks = current_app.config["HUB_MODEL"].virsh_netlist()
db_hosts = get_model().list_hosts_with_networks(None)
virt_vms_by_id = dict()
for kv in db_hosts.items():
host_id = kv[0]
value = kv[1]
for network in value['networks']:
for vm in db_vms:
if vm["id"] not in db_vms_by_id:
# TODO
raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
else:
db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
virt_vms = current_app.config["HUB_MODEL"].get_vm_()
def ensure_vms_and_db_are_synced():
# Now creating the capsuls running status ui
#
for vm in db_vms:
db_ids_dict[vm['id']] = vm['desired_state']
for vm in virt_vms:
virt_ids_dict[vm['id']] = vm['desired_state']
errors = list()
for id in db_ids_dict:
if id not in virt_ids_dict:
errors.append(f"{id} is in the database but not in the virtualization model")
elif db_ids_dict[id] != virt_ids_dict[id]:
errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
for id in virt_ids_dict:
if id not in db_ids_dict:
errors.append(f"{id} is in the virtualization model but not in the database")
if len(errors) > 0:
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
for error in errors:
current_app.logger.info(f"cron_task: {error}.")
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Capsul Consistency Check Failed",
sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(errors),
recipients=email_addresses
)
)

View File

@ -27,12 +27,12 @@ def make_capsul_id():
letters_n_nummers = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
return f"capsul-{letters_n_nummers}"
def double_check_capsul_address(id, ipv4, get_ssh_host_keys):
def double_check_capsul_address(id, get_ssh_host_keys):
try:
result = current_app.config["HUB_MODEL"].get(id, get_ssh_host_keys)
if result != None and result.ipv4 != None and result.ipv4 != ipv4:
ipv4 = result.ipv4
get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4)
# if result != None and result.ipv4 != None and result.ipv4 != ipv4:
# ipv4 = result.ipv4
# get_model().update_vm_ip(email=session["account"], id=id, ipv4=result.ipv4)
if result != None and result.ssh_host_keys != None and get_ssh_host_keys:
get_model().update_vm_ssh_host_keys(email=session["account"], id=id, ssh_host_keys=result.ssh_host_keys)
@ -59,7 +59,7 @@ def index():
# for now we are going to check the IP according to the virt model
# on every request. this could be done by a background job and cached later on...
for vm in vms:
result = double_check_capsul_address(vm["id"], vm["ipv4"], False)
result = double_check_capsul_address(vm["id"], False)
if result is not None:
vm["ipv4"] = result.ipv4
vm["state"] = result.state
@ -167,7 +167,7 @@ def detail(id):
else:
needs_ssh_host_keys = "ssh_host_keys" not in vm or len(vm["ssh_host_keys"]) == 0
vm_from_virt_model = double_check_capsul_address(vm["id"], vm["ipv4"], needs_ssh_host_keys)
vm_from_virt_model = double_check_capsul_address(vm["id"], needs_ssh_host_keys)
if vm_from_virt_model is not None:
vm["ipv4"] = vm_from_virt_model.ipv4

View File

@ -50,7 +50,7 @@ def init_app(app, is_running_server):
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 20
desiredSchemaVersion = 21
cursor = connection.cursor()

View File

@ -86,9 +86,9 @@ class DBModel:
return hosts
def all_non_deleted_vm_ids(self):
self.cursor.execute("SELECT id FROM vms WHERE deleted IS NULL")
return list(map(lambda x: x[0], self.cursor.fetchall()))
def all_vm_ids_with_desired_state(self):
self.cursor.execute("SELECT id, desired_state FROM vms WHERE deleted IS NULL")
return list(map(lambda x: {"id": x[0], "desired_state": x[1]}, self.cursor.fetchall()))
def operating_systems_dict(self):
self.cursor.execute("SELECT id, template_image_file_name, description FROM os_images WHERE deprecated = FALSE")
@ -332,7 +332,7 @@ class DBModel:
def list_hosts_with_networks(self, host_id: str):
query = """
SELECT hosts.id, hosts.last_health_check, host_network.network_name,
SELECT hosts.id, hosts.last_health_check, host_network.network_name, host_network.virtual_bridge_name,
host_network.public_ipv4_cidr_block, host_network.public_ipv4_first_usable_ip, host_network.public_ipv4_last_usable_ip
FROM hosts
JOIN host_network ON host_network.host = hosts.id
@ -355,9 +355,10 @@ class DBModel:
hosts[row[0]]["networks"].append(dict(
network_name=row[2],
public_ipv4_cidr_block=row[3],
public_ipv4_first_usable_ip=row[4],
public_ipv4_last_usable_ip=row[5]
virtual_bridge_name=row[3],
public_ipv4_cidr_block=row[4],
public_ipv4_first_usable_ip=row[5],
public_ipv4_last_usable_ip=row[6]
))
return hosts

View File

@ -37,8 +37,8 @@ class MockHub(VirtualizationInterface):
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4)
def list_ids(self) -> list:
return get_model().all_non_deleted_vm_ids()
def get_all_by_host_and_network(self) -> dict:
return get_model().non_deleted_vms_by_host_and_network()
def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
@ -164,29 +164,24 @@ class CapsulFlaskHub(VirtualizationInterface):
return None
def list_ids(self) -> list:
def get_all_by_host_and_network(self) -> dict:
online_hosts = get_model().get_online_hosts()
payload = json.dumps(dict(type="list_ids"))
payload = json.dumps(dict(type="get_all_by_host_and_network"))
results = self.synchronous_operation(online_hosts, None, payload)
to_return = []
to_return = dict()
for i in range(len(results)):
host = online_hosts[i]
result = results[i]
try:
result_body = json.loads(result.body)
if isinstance(result_body, dict) and 'ids' in result_body and isinstance(result_body['ids'], list):
all_valid = True
for id in result_body['ids']:
try:
validate_capsul_id(id)
to_return.append(id)
except:
all_valid = False
if not all_valid:
current_app.logger.error(f"""error reading ids for list_ids operation, host {host.id}""")
has_host = isinstance(result_body, dict) and host.id in result_body and isinstance(result_body[host.id], dict)
has_networks = has_host and 'networks' in result_body[host.id] and isinstance(result_body[host.id]['networks'], dict)
if has_host and has_networks:
to_return[host.id] = result_body[host.id]['networks']
else:
result_json_string = json.dumps({"error_message": "invalid response, missing 'ids' list"})
current_app.logger.error(f"""missing 'ids' list for list_ids operation, host {host.id}""")
# result_json_string = json.dumps({"error_message": "invalid response, missing 'networks' list"})
current_app.logger.error(f"""missing 'networks' list for get_all_by_host_and_network operation, host {host.id}""")
except:
# no need to do anything here since if it cant be parsed then generic_operation will handle it.
pass

View File

@ -0,0 +1,7 @@
ALTER TABLE vms DROP COLUMN desired_state;
UPDATE schemaversion SET version = 20;

View File

@ -0,0 +1,7 @@
ALTER TABLE vms ADD COLUMN desired_state TEXT DEFAULT 'running';
UPDATE schemaversion SET version = 21;

View File

@ -31,7 +31,7 @@ class VirtualizationInterface:
def get(self, id: str) -> VirtualMachine:
pass
def list_ids(self) -> list:
def get_all_by_host_and_network(self) -> dict:
pass
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory: int, ssh_authorized_keys: list):

View File

@ -1,3 +0,0 @@
#!/bin/sh
virsh list --all | grep running | grep -v ' Id' | grep -v -- '----' | awk '{print $2}' | sort

View File

@ -8,15 +8,15 @@ if echo "$ip_address" | grep -vqE '^([0-9]{1,3}\.){3}[0-9]{1,3}$'; then
fi
printf '['
DELIMITER=""
delimiter=""
ssh-keyscan "$ip_address" 2>/dev/null | while read -r line; do
if echo "$line" | grep -qE "^$ip_address"' +(ssh|ecdsa)-[0-9A-Za-z+/_=@. -]+$'; then
KEY_CONTENT="$(echo "$line" | awk '{ print $2 " " $3 }')"
FINGERPRINT_OUTPUT="$(echo "$KEY_CONTENT" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')"
SHA256_HASH="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $1 }')"
KEY_TYPE="$(echo "$FINGERPRINT_OUTPUT" | awk '{ print $2 }')"
printf '%s\n {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$DELIMITER" "$KEY_TYPE" "$KEY_CONTENT" "$SHA256_HASH"
DELIMITER=","
key_content="$(echo "$line" | awk '{ print $2 " " $3 }')"
fingerprint_output="$(echo "$key_content" | ssh-keygen -l -E sha256 -f - | sed -E 's/^[0-9]+ SHA256:([0-9A-Za-z+/-]+) .+ \(([A-Z0-9]+)\)$/\1 \2/g')"
sha256_hash="$(echo "$fingerprint_output" | awk '{ print $1 }')"
key_type="$(echo "$fingerprint_output" | awk '{ print $2 }')"
printf '%s\n {"key_type":"%s", "content":"%s", "sha256":"%s"}' "$delimiter" "$key_type" "$key_content" "$sha256_hash"
delimiter=","
fi
done
printf '\n]\n'

View File

@ -0,0 +1,13 @@
#!/bin/sh
printf '['
delimiter=""
virsh list --all | while read -r line; do
if echo "$line" | grep -qE '(running)|(shut off)'; then
capsul_id="$(echo "$line" | awk '{ print $2 }')"
capsul_state="$(echo "$line" | sed -E 's/.*((running)|(shut off))\w*/\1/')"
printf '%s\n {"id":"%s", "state":"%s"}' "$delimiter" "$capsul_id" "$capsul_state"
delimiter=","
fi
done
printf '\n]\n'

View File

@ -0,0 +1,3 @@
#!/bin/sh
virsh net-list --all | tail -n +3 | awk '{ print $1 }'

View File

@ -49,7 +49,7 @@ def operation_impl(operation_id: int):
handlers = {
"capacity_avaliable": handle_capacity_avaliable,
"get": handle_get,
"list_ids": handle_list_ids,
"get_all_by_host_and_network": handle_get_all_by_host_and_network,
"create": handle_create,
"destroy": handle_destroy,
"vm_state_command": handle_vm_state_command,
@ -96,8 +96,8 @@ def handle_get(operation_id, request_body):
return jsonify(dict(assignment_status="assigned", id=vm.id, host=vm.host, state=vm.state, ipv4=vm.ipv4, ipv6=vm.ipv6, ssh_host_keys=vm.ssh_host_keys))
def handle_list_ids(operation_id, request_body):
return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].list_ids()))
def handle_get_all_by_host_and_network(operation_id, request_body):
return jsonify(dict(assignment_status="assigned", ids=current_app.config['SPOKE_MODEL'].get_all_by_host_and_network()))
def handle_create(operation_id, request_body):
if not operation_id:

View File

@ -38,8 +38,8 @@ class MockSpoke(VirtualizationInterface):
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
def list_ids(self) -> list:
return get_model().all_non_deleted_vm_ids()
def get_all_by_host_and_network(self) -> dict:
return get_model().non_deleted_vms_by_host_and_network(None)
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
validate_capsul_id(id)
@ -133,10 +133,11 @@ class ShellScriptSpoke(VirtualizationInterface):
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], state=state, ipv4=ipaddr)
def list_ids(self) -> list:
completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True)
def get_all_by_host_and_network(self) -> list:
# TODO implement this
completedProcess = run([join(current_app.root_path, 'shell_scripts/virsh-list.sh')], capture_output=True)
self.validate_completed_process(completedProcess)
return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() ))
return json.loads(completedProcess.stdout.decode("utf-8"))
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
validate_capsul_id(id)