autonomic-cooperative
/
capsul-flask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

fixing capsul creation after I broke it with the pre-allocated IP 

address changes
This commit is contained in:
forest 2021-07-11 12:18:58 -05:00
parent a2f2e744e4
commit fcbea1e29b
7 changed files with 83 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
capsulflask/admin.py
View File

@ -54,7 +54,7 @@ def index():
if network['network_name'] in vms_by_host_and_network[host_id]:
for vm in vms_by_host_and_network[host_id][network['network_name']]:
ip_address_int = int(ipaddress.ip_address(vm['public_ipv4']))
if network_start_int < ip_address_int and ip_address_int < network_end_int:
if network_start_int <= ip_address_int and ip_address_int <= network_end_int:
allocation = f"{host_id}_{network['network_name']}_{len(network['allocations'])}"
inline_styles.append(
f"""

13
capsulflask/console.py
View File

@ -245,20 +245,17 @@ def create():
if len(errors) == 0:
id = make_capsul_id()
get_model().create_vm(
email=session["account"],
id=id,
size=size,
os=os,
ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys))
)
# we can't create the vm record in the DB yet because its IP address needs to be allocated first.
# so it will be created when the allocation happens inside the hub_api.
current_app.config["HUB_MODEL"].create(
email = session["account"],
id=id,
os=os,
size=size,
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys))
ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys))
)
return redirect(f"{url_for('console.index')}?created={id}")

26
capsulflask/db_model.py
View File

@ -1,4 +1,6 @@
import re
# I was never able to get this type hinting to work correctly
# from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
from nanoid import generate
@ -61,7 +63,13 @@ class DBModel:
if host_id is None:
self.cursor.execute(query)
else:
self.cursor.execute(f"{query} AND host = %s", (host_id))
if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")
# I kept getting "TypeError: not all arguments converted during string formatting"
# when I was trying to mix python string templating with psycopg2 safe parameter passing.
# so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
self.cursor.execute(f"{query} AND host = '{host_id}'")
hosts = dict()
for row in self.cursor.fetchall():
@ -150,12 +158,12 @@ class DBModel:
)
self.connection.commit()
def create_vm(self, email, id, size, os, ssh_authorized_keys):
def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys):
self.cursor.execute("""
INSERT INTO vms (email, id, size, os)
VALUES (%s, %s, %s, %s)
INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(email, id, size, os)
(email, id, size, os, host, network_name, public_ipv4)
)
for ssh_authorized_key in ssh_authorized_keys:
@ -330,7 +338,13 @@ class DBModel:
if host_id is None:
self.cursor.execute(query)
else:
self.cursor.execute(f"{query} WHERE hosts.id = %s", (host_id))
if not re.match(r"^[a-zA-Z0-9_-]+$", host_id):
raise ValueError(f"host_id \"{host_id}\" must match \"^[a-zA-Z0-9_-]+\"")
# I kept getting "TypeError: not all arguments converted during string formatting"
# when I was trying to mix python query string templating with psycopg2 safe parameter passing.
# so i just did all of it in python and check the user-provided data for safety myself (no sql injection).
self.cursor.execute(f"{query} WHERE hosts.id = '{host_id}'")
hosts = dict()
for row in self.cursor.fetchall():

37
capsulflask/hub_api.py
View File

@ -54,9 +54,14 @@ def claim_operation(operation_id: int, host_id: str):
current_app.logger.error(error_message)
return abort(404, error_message)
# right now if there is a can_claim_handler there needs to be a corresponding on_claimed_handler.
# this is sole-ly due to laziness in error handling.
can_claim_handlers = {
"create": can_claim_create,
}
on_claimed_handlers = {
"create": on_create_claimed,
}
error_message = ""
payload = None
payload_is_dict = False
@ -77,7 +82,7 @@ def claim_operation(operation_id: int, host_id: str):
except:
error_message = "could not parse payload as json"
if error_message is not "":
if error_message != "":
error_message = f"{host_id} can't claim operation {operation_id} because {error_message}"
current_app.logger.error(error_message)
return abort(400, error_message)
@ -87,20 +92,22 @@ def claim_operation(operation_id: int, host_id: str):
# invoke the appropriate can_claim_handler for this operation type
result_tuple = can_claim_handlers[payload['type']](payload, host_id)
payload_json = result_tuple[0]
modified_payload = result_tuple[0]
error_message = result_tuple[1]
if error_message is not "":
if error_message != "":
error_message = f"{host_id} can't claim operation {operation_id} because {error_message}"
current_app.logger.error(error_message)
return abort(400, error_message)
claimed = get_model().claim_operation(operation_id, host_id)
if claimed:
get_model().update_operation(operation_id, payload_json)
modified_payload_json = json.dumps(modified_payload)
get_model().update_operation(operation_id, modified_payload_json)
on_claimed_handlers[payload['type']](modified_payload, host_id)
response = make_response(payload_json)
response.header.set("Content-Type", "application/json")
response = make_response(modified_payload_json)
response.headers.set("Content-Type", "application/json")
return response
else:
@ -116,7 +123,7 @@ def can_claim_create(payload, host_id) -> (str, str):
if host_id not in hosts:
return "", f"the host \"{host_id}\" does not appear to have any networks."
networks = hosts[host_id].networks
networks = hosts[host_id]['networks']
vms_by_host_and_network = get_model().non_deleted_vms_by_host_and_network(host_id)
@ -151,6 +158,18 @@ def can_claim_create(payload, host_id) -> (str, str):
return "", f"host \"{host_id}\" does not have any avaliable IP addresses on any of its networks."
payload["network_name"] = allocated_network_name
payload["public_ipv4_address"] = allocated_ipv4_address
payload["public_ipv4"] = allocated_ipv4_address
return json.dumps(payload), ""
return payload, ""
def on_create_claimed(payload, host_id):
get_model().create_vm(
email=payload['email'],
id=payload['id'],
size=payload['size'],
os=payload['os'],
host=host_id,
network_name=payload['network_name'],
public_ipv4=payload['public_ipv4'],
ssh_authorized_keys=list(map(lambda x: x["name"], payload['ssh_authorized_keys'])),
)

6
capsulflask/hub_model.py
View File

@ -36,7 +36,7 @@ class MockHub(VirtualizationInterface):
def list_ids(self) -> list:
return get_model().all_non_deleted_vm_ids()
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
current_app.logger.info(f"mock create: {id} for {email}")
sleep(1)
@ -180,7 +180,7 @@ class CapsulFlaskHub(VirtualizationInterface):
return to_return
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
online_hosts = get_model().get_online_hosts()
#current_app.logger.debug(f"hub_model.create(): ${len(online_hosts)} hosts")
@ -188,6 +188,8 @@ class CapsulFlaskHub(VirtualizationInterface):
type="create",
email=email,
id=id,
os=os,
size=size,
template_image_file_name=template_image_file_name,
vcpus=vcpus,
memory_mb=memory_mb,

17
capsulflask/spoke_api.py
View File

@ -104,7 +104,7 @@ def handle_create(operation_id, request_body):
current_app.logger.error(f"/hosts/operation returned 400: operation_id is required for create ")
return abort(400, f"bad request; operation_id is required. try POST /spoke/operation/<id>")
parameters = ["email", "id", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"]
parameters = ["email", "id", "os", "size", "template_image_file_name", "vcpus", "memory_mb", "ssh_authorized_keys"]
error_message = ""
for parameter in parameters:
if parameter not in request_body:
@ -128,16 +128,15 @@ def handle_create(operation_id, request_body):
return abort(503, f"hub at '{url}' returned 200, but did not return assignment_info json object")
if 'network_name' not in assignment_info:
return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include network_name")
if 'public_ipv4_address' not in assignment_info:
return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include public_ipv4_address")
if 'public_ipv4' not in assignment_info:
return abort(503, f"hub at '{url}' returned 200, but the returned assignment_info object did not include public_ipv4")
assignment_status = "assigned"
request_body['network_name'] = assignment_info['network_name']
request_body['public_ipv4_address'] = assignment_info['public_ipv4_address']
request_body['public_ipv4'] = assignment_info['public_ipv4']
except:
return abort(503, f"hub at '{url}' returned 200, but did not return valid json")
assignment_status = "assigned"
elif result.status_code == 409:
assignment_status = "assigned_to_other_host"
else:
@ -152,9 +151,9 @@ def handle_create(operation_id, request_body):
template_image_file_name=request_body['template_image_file_name'],
vcpus=request_body['vcpus'],
memory_mb=request_body['memory_mb'],
ssh_authorized_keys=request_body['ssh_authorized_keys'],
ssh_authorized_keys=list(map(lambda x: x['content'], request_body['ssh_authorized_keys'])),
network_name=request_body['network_name'],
public_ipv4_address=request_body['public_ipv4_address'],
public_ipv4=request_body['public_ipv4'],
)
except:
error_message = my_exec_info_message(sys.exc_info())
@ -165,7 +164,7 @@ def handle_create(operation_id, request_body):
params= f"{params} memory_mb='{request_body['memory_mb'] if 'memory_mb' in request_body else 'KeyError'}', "
params= f"{params} ssh_authorized_keys='{request_body['ssh_authorized_keys'] if 'ssh_authorized_keys' in request_body else 'KeyError'}', "
params= f"{params} network_name='{request_body['network_name'] if 'network_name' in request_body else 'KeyError'}', "
params= f"{params} public_ipv4_address='{request_body['public_ipv4_address'] if 'public_ipv4_address' in request_body else 'KeyError'}', "
params= f"{params} public_ipv4='{request_body['public_ipv4'] if 'public_ipv4' in request_body else 'KeyError'}', "
current_app.logger.error(f"spoke_model.create({params}) failed: {error_message}")

25
capsulflask/spoke_model.py
View File

@ -14,28 +14,37 @@ from capsulflask.shared import VirtualizationInterface, VirtualMachine, validate
class MockSpoke(VirtualizationInterface):
def __init__(self):
self.capsuls = dict()
def capacity_avaliable(self, additional_ram_bytes):
return True
def get(self, id, get_ssh_host_keys):
validate_capsul_id(id)
ipv4 = "1.1.1.1"
if id in self.capsuls:
ipv4 = self.capsuls[id]['public_ipv4']
if get_ssh_host_keys:
ssh_host_keys = json.loads("""[
{"key_type":"ED25519", "content":"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN8cna0zeKSKl/r8whdn/KmDWhdzuWRVV0GaKIM+eshh", "sha256":"V4X2apAF6btGAfS45gmpldknoDX0ipJ5c6DLfZR2ttQ"},
{"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"},
{"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"}
]""")
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running", ssh_host_keys=ssh_host_keys)
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running", ssh_host_keys=ssh_host_keys)
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", state="running")
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
def list_ids(self) -> list:
return get_model().all_non_deleted_vm_ids()
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
validate_capsul_id(id)
current_app.logger.info(f"mock create: {id} for {email}")
self.capsuls[id] = dict(email=email, id=id, network_name=network_name, public_ipv4=public_ipv4)
sleep(1)
def destroy(self, email: str, id: str):
@ -129,7 +138,7 @@ class ShellScriptSpoke(VirtualizationInterface):
self.validate_completed_process(completedProcess)
return list(map(lambda x: x.decode("utf-8"), completedProcess.stdout.splitlines() ))
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4_address: str):
def create(self, email: str, id: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list, network_name: str, public_ipv4: str):
validate_capsul_id(id)
if not re.match(r"^[a-zA-Z0-9/_.-]+$", template_image_file_name):
@ -148,8 +157,8 @@ class ShellScriptSpoke(VirtualizationInterface):
if not re.match(r"^[a-zA-Z0-9_-]+$", network_name):
raise ValueError(f"network_name \"{network_name}\" must match \"^[a-zA-Z0-9_-]+\"")
if not re.match(r"^[0-9.]+$", public_ipv4_address):
raise ValueError(f"public_ipv4_address \"{public_ipv4_address}\" must match \"^[0-9.]+$\"")
if not re.match(r"^[0-9.]+$", public_ipv4):
raise ValueError(f"public_ipv4 \"{public_ipv4}\" must match \"^[0-9.]+$\"")
ssh_keys_string = "\n".join(ssh_authorized_keys)
@ -161,7 +170,7 @@ class ShellScriptSpoke(VirtualizationInterface):
str(memory_mb),
ssh_keys_string,
network_name,
public_ipv4_address
public_ipv4
], capture_output=True)
self.validate_completed_process(completedProcess, email)
@ -175,7 +184,7 @@ class ShellScriptSpoke(VirtualizationInterface):
memory={str(memory_mb)}
ssh_authorized_keys={ssh_keys_string}
network_name={network_name}
public_ipv4_address={public_ipv4_address}
public_ipv4={public_ipv4}
"""
if not status == "success":