From fcbea1e29b11e6c474bbaa7c5a9990e4254e9731 Mon Sep 17 00:00:00 2001 From: forest Date: Sun, 11 Jul 2021 12:18:58 -0500 Subject: [PATCH] fixing capsul creation after I broke it with the pre-allocated IP address changes --- capsulflask/admin.py | 2 +- capsulflask/console.py | 13 +++++-------- capsulflask/db_model.py | 26 ++++++++++++++++++++------ capsulflask/hub_api.py | 37 ++++++++++++++++++++++++++++--------- capsulflask/hub_model.py | 6 ++++-- capsulflask/spoke_api.py | 17 ++++++++--------- capsulflask/spoke_model.py | 25 +++++++++++++++++-------- 7 files changed, 83 insertions(+), 43 deletions(-) diff --git a/capsulflask/admin.py b/capsulflask/admin.py index 22f94d6..511c94d 100644 --- a/capsulflask/admin.py +++ b/capsulflask/admin.py @@ -54,7 +54,7 @@ def index(): if network['network_name'] in vms_by_host_and_network[host_id]: for vm in vms_by_host_and_network[host_id][network['network_name']]: ip_address_int = int(ipaddress.ip_address(vm['public_ipv4'])) - if network_start_int < ip_address_int and ip_address_int < network_end_int: + if network_start_int <= ip_address_int and ip_address_int <= network_end_int: allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}" inline_styles.append( f""" diff --git a/capsulflask/console.py b/capsulflask/console.py index 55e0ffa..e97b805 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -245,20 +245,17 @@ def create(): if len(errors) == 0: id = make_capsul_id() - get_model().create_vm( - email=session["account"], - id=id, - size=size, - os=os, - ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys)) - ) + # we can't create the vm record in the DB yet because its IP address needs to be allocated first. + # so it will be created when the allocation happens inside the hub_api. current_app.config["HUB_MODEL"].create( email = session["account"], id=id, + os=os, + size=size, template_image_file_name=operating_systems[os]['template_image_file_name'], vcpus=vm_sizes[size]['vcpus'], memory_mb=vm_sizes[size]['memory_mb'], - ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys)) + ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys)) ) return redirect(f"{url_for('console.index')}?created={id}") diff --git a/capsulflask/db_model.py b/capsulflask/db_model.py index d7df878..a6ef58d 100644 --- a/capsulflask/db_model.py +++ b/capsulflask/db_model.py @@ -1,4 +1,6 @@ +import re + # I was never able to get this type hinting to work correctly # from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor from nanoid import generate @@ -61,7 +63,13 @@ class DBModel: if host_id is None: self.cursor.execute(query) else: - self.cursor.execute(f"{query} AND host = %s", (host_id)) + if not re.match(r"^[a-zA-Z0-9_-]+$", host_id): + raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"") + + # I kept getting "TypeError: not all arguments converted during string formatting" + # when I was trying to mix python string templating with psycopg2 safe parameter passing. + # so i just did all of it in python and check the user-provided data for safety myself (no sql injection). + self.cursor.execute(f"{query} AND host = '{host_id}'") hosts = dict() for row in self.cursor.fetchall(): @@ -150,12 +158,12 @@ class DBModel: ) self.connection.commit() - def create_vm(self, email, id, size, os, ssh_authorized_keys): + def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys): self.cursor.execute(""" - INSERT INTO vms (email, id, size, os) - VALUES (%s, %s, %s, %s) + INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4) + VALUES (%s, %s, %s, %s, %s, %s, %s) """, - (email, id, size, os) + (email, id, size, os, host, network_name, public_ipv4) ) for ssh_authorized_key in ssh_authorized_keys: @@ -330,7 +338,13 @@ class DBModel: if host_id is None: self.cursor.execute(query) else: - self.cursor.execute(f"{query} WHERE hosts.id = %s", (host_id)) + if not re.match(r"^[a-zA-Z0-9_-]+$", host_id): + raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"") + + # I kept getting "TypeError: not all arguments converted during string formatting" + # when I was trying to mix python query string templating with psycopg2 safe parameter passing. + # so i just did all of it in python and check the user-provided data for safety myself (no sql injection). + self.cursor.execute(f"{query} WHERE hosts.id = '{host_id}'") hosts = dict() for row in self.cursor.fetchall(): diff --git a/capsulflask/hub_api.py b/capsulflask/hub_api.py index 11b91f9..4d7e9cb 100644 --- a/capsulflask/hub_api.py +++ b/capsulflask/hub_api.py @@ -54,9 +54,14 @@ def claim_operation(operation_id: int, host_id: str): current_app.logger.error(error_message) return abort(404, error_message) + # right now if there is a can_claim_handler there needs to be a corresponding on_claimed_handler. + # this is sole-ly due to laziness in error handling. can_claim_handlers = { "create": can_claim_create, } + on_claimed_handlers = { + "create": on_create_claimed, + } error_message = "" payload = None payload_is_dict = False @@ -77,7 +82,7 @@ def claim_operation(operation_id: int, host_id: str): except: error_message = "could not parse payload as json" - if error_message is not "": + if error_message != "": error_message = f"{host_id} can't claim operation {operation_id} because {error_message}" current_app.logger.error(error_message) return abort(400, error_message) @@ -87,20 +92,22 @@ def claim_operation(operation_id: int, host_id: str): # invoke the appropriate can_claim_handler for this operation type result_tuple = can_claim_handlers[payload['type']](payload, host_id) - payload_json = result_tuple[0] + modified_payload = result_tuple[0] error_message = result_tuple[1] - if error_message is not "": + if error_message != "": error_message = f"{host_id} can't claim operation {operation_id} because {error_message}" current_app.logger.error(error_message) return abort(400, error_message) claimed = get_model().claim_operation(operation_id, host_id) if claimed: - get_model().update_operation(operation_id, payload_json) + modified_payload_json = json.dumps(modified_payload) + get_model().update_operation(operation_id, modified_payload_json) + on_claimed_handlers[payload['type']](modified_payload, host_id) - response = make_response(payload_json) - response.header.set("Content-Type", "application/json") + response = make_response(modified_payload_json) + response.headers.set("Content-Type", "application/json") return response else: @@ -116,7 +123,7 @@ def can_claim_create(payload, host_id) -> (str, str): if host_id not in hosts: return "", f"the host \"{host_id}\" does not appear to have any networks." - networks = hosts[host_id].networks + networks = hosts[host_id]['networks'] vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(host_id) @@ -151,6 +158,18 @@ def can_claim_create(payload, host_id) -> (str, str): return "", f"host \"{host_id}\" does not have any avaliable IP addresses on any of its networks." payload["network_name"] = allocated_network_name - payload["public_ipv4_address"] = allocated_ipv4_address + payload["public_ipv4"] = allocated_ipv4_address - return json.dumps(payload), "" \ No newline at end of file + return payload, "" + +def on_create_claimed(payload, host_id): + get_model().create_vm( + email=payload['email'], + id=payload['id'], + size=payload['size'], + os=payload['os'], + host=host_id, + network_name=payload['network_name'], + public_ipv4=payload['public_ipv4'], + ssh_authorized_keys=list(map(lambda x: x["name"], payload['ssh_authorized_keys'])), + ) \ No newline at end of file diff --git a/capsulflask/hub_model.py b/capsulflask/hub_model.py index b9fd1ee..ac2a865 100644 --- a/capsulflask/hub_model.py +++ b/capsulflask/hub_model.py @@ -36,7 +36,7 @@ class MockHub(VirtualizationInterface): def list_ids(self) -> list: return get_model().all_non_deleted_vm_ids() - def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): + def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) current_app.logger.info(f"mock create: {id} for {email}") sleep(1) @@ -180,7 +180,7 @@ class CapsulFlaskHub(VirtualizationInterface): return to_return - def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): + def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) online_hosts = get_model().get_online_hosts() #current_app.logger.debug(f"hub_model.create(): ${len(online_hosts)} hosts") @@ -188,6 +188,8 @@ class CapsulFlaskHub(VirtualizationInterface): type="create", email=email, id=id, + os=os, + size=size, template_image_file_name=template_image_file_name, vcpus=vcpus, memory_mb=memory_mb, diff --git a/capsulflask/spoke_api.py b/capsulflask/spoke_api.py index 16be975..3ca86bc 100644 --- a/capsulflask/spoke_api.py +++ b/capsulflask/spoke_api.py @@ -104,7 +104,7 @@ def handle_create(operation_id, request_body): current_app.logger.error(f"/hosts/operation returned 400: operation_id is required for create ") return abort(400, f"bad request; operation_id is required. try POST /spoke/operation/") - parameters = ["email", "id", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"] + parameters = ["email", "id", "os", "size", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"] error_message = "" for parameter in parameters: if parameter not in request_body: @@ -128,16 +128,15 @@ def handle_create(operation_id, request_body): return abort(503, f"hub at '{url}' returned 200, but did not return assignment_info json object") if 'network_name' not in assignment_info: return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include network_name") - if 'public_ipv4_address' not in assignment_info: - return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include public_ipv4_address") + if 'public_ipv4' not in assignment_info: + return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include public_ipv4") + assignment_status = "assigned" request_body['network_name'] = assignment_info['network_name'] - request_body['public_ipv4_address'] = assignment_info['public_ipv4_address'] + request_body['public_ipv4'] = assignment_info['public_ipv4'] except: return abort(503, f"hub at '{url}' returned 200, but did not return valid json") - assignment_status = "assigned" - elif result.status_code == 409: assignment_status = "assigned_to_other_host" else: @@ -152,9 +151,9 @@ def handle_create(operation_id, request_body): template_image_file_name=request_body['template_image_file_name'], vcpus=request_body['vcpus'], memory_mb=request_body['memory_mb'], - ssh_authorized_keys=request_body['ssh_authorized_keys'], + ssh_authorized_keys=list(map(lambda x: x['content'], request_body['ssh_authorized_keys'])), network_name=request_body['network_name'], - public_ipv4_address=request_body['public_ipv4_address'], + public_ipv4=request_body['public_ipv4'], ) except: error_message = my_exec_info_message(sys.exc_info()) @@ -165,7 +164,7 @@ def handle_create(operation_id, request_body): params= f"{params} memory_mb='{request_body['memory_mb'] if 'memory_mb' in request_body else 'KeyError'}', " params= f"{params} ssh_authorized_keys='{request_body['ssh_authorized_keys'] if 'ssh_authorized_keys' in request_body else 'KeyError'}', " params= f"{params} network_name='{request_body['network_name'] if 'network_name' in request_body else 'KeyError'}', " - params= f"{params} public_ipv4_address='{request_body['public_ipv4_address'] if 'public_ipv4_address' in request_body else 'KeyError'}', " + params= f"{params} public_ipv4='{request_body['public_ipv4'] if 'public_ipv4' in request_body else 'KeyError'}', " current_app.logger.error(f"spoke_model.create({params}) failed: {error_message}") diff --git a/capsulflask/spoke_model.py b/capsulflask/spoke_model.py index a11270d..9ab0d3a 100644 --- a/capsulflask/spoke_model.py +++ b/capsulflask/spoke_model.py @@ -14,28 +14,37 @@ from capsulflask.shared import VirtualizationInterface, VirtualMachine, validate class MockSpoke(VirtualizationInterface): + + def __init__(self): + self.capsuls = dict() + def capacity_avaliable(self, additional_ram_bytes): return True def get(self, id, get_ssh_host_keys): validate_capsul_id(id) + ipv4 = "1.1.1.1" + if id in self.capsuls: + ipv4 = self.capsuls[id]['public_ipv4'] + if get_ssh_host_keys: ssh_host_keys = json.loads("""[ {"key_type":"ED25519", "content":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN8cna0zeKSKl/r8whdn/KmDWhdzuWRVV0GaKIM+eshh", "sha256":"V4X2apAF6btGAfS45gmpldknoDX0ipJ5c6DLfZR2ttQ"}, {"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"}, {"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"} ]""") - return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running", ssh_host_keys=ssh_host_keys) + return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running", ssh_host_keys=ssh_host_keys) - return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running") + return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running") def list_ids(self) -> list: return get_model().all_non_deleted_vm_ids() - def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): + def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str): validate_capsul_id(id) current_app.logger.info(f"mock create: {id} for {email}") + self.capsuls[id] = dict(email=email, id=id, network_name=network_name, public_ipv4=public_ipv4) sleep(1) def destroy(self, email: str, id: str): @@ -129,7 +138,7 @@ class ShellScriptSpoke(VirtualizationInterface): self.validate_completed_process(completedProcess) return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() )) - def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4_address: str): + def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str): validate_capsul_id(id) if not re.match(r"^[a-zA-Z0-9/_.-]+$", template_image_file_name): @@ -148,8 +157,8 @@ class ShellScriptSpoke(VirtualizationInterface): if not re.match(r"^[a-zA-Z0-9_-]+$", network_name): raise ValueError(f"network_name \"{network_name}\" must match \"^[a-zA-Z0-9_-]+\"") - if not re.match(r"^[0-9.]+$", public_ipv4_address): - raise ValueError(f"public_ipv4_address \"{public_ipv4_address}\" must match \"^[0-9.]+$\"") + if not re.match(r"^[0-9.]+$", public_ipv4): + raise ValueError(f"public_ipv4 \"{public_ipv4}\" must match \"^[0-9.]+$\"") ssh_keys_string = "\n".join(ssh_authorized_keys) @@ -161,7 +170,7 @@ class ShellScriptSpoke(VirtualizationInterface): str(memory_mb), ssh_keys_string, network_name, - public_ipv4_address + public_ipv4 ], capture_output=True) self.validate_completed_process(completedProcess, email) @@ -175,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface): memory={str(memory_mb)} ssh_authorized_keys={ssh_keys_string} network_name={network_name} - public_ipv4_address={public_ipv4_address} + public_ipv4={public_ipv4} """ if not status == "success":