capacity limiter and general cleanup

This commit is contained in:
forest 2020-05-13 13:56:43 -05:00
parent a43b3b4e1e
commit 403506a0b0
10 changed files with 121 additions and 50 deletions

View File

@ -75,7 +75,7 @@ def detail(id):
vm["ipv4"] = double_check_capsul_address(vm["id"], vm["ipv4"]) vm["ipv4"] = double_check_capsul_address(vm["id"], vm["ipv4"])
vm["created"] = vm['created'].strftime("%b %d %Y %H:%M") vm["created"] = vm['created'].strftime("%b %d %Y %H:%M")
vm["ssh_public_keys"] = ", ".join(vm["ssh_public_keys"]) if len(vm["ssh_public_keys"]) > 0 else "<deleted>" vm["ssh_public_keys"] = ", ".join(vm["ssh_public_keys"]) if len(vm["ssh_public_keys"]) > 0 else "<missing>"
return render_template( return render_template(
"capsul-detail.html", "capsul-detail.html",
@ -92,6 +92,7 @@ def create():
operating_systems = get_model().operating_systems_dict() operating_systems = get_model().operating_systems_dict()
ssh_public_keys = get_model().list_ssh_public_keys_for_account(session["account"]) ssh_public_keys = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance() account_balance = get_account_balance()
capacity_avaliable = current_app.config["VIRTUALIZATION_MODEL"].capacity_avaliable(512*1024*1024)
errors = list() errors = list()
created_os = None created_os = None
@ -130,6 +131,13 @@ def create():
if len(posted_keys) == 0: if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required") errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["VIRTUALIZATION_MODEL"].capacity_avaliable(vm_sizes[size]['memory_mb']*1024*1024)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0: if len(errors) == 0:
id = makeCapsulId() id = makeCapsulId()
get_model().create_vm( get_model().create_vm(
@ -157,9 +165,13 @@ def create():
for error in errors: for error in errors:
flash(error) flash(error)
if not capacity_avaliable:
print(f"when capsul capacity is restored, send an email to {session['account']}")
return render_template( return render_template(
"create-capsul.html", "create-capsul.html",
created_os=created_os, created_os=created_os,
capacity_avaliable=capacity_avaliable,
account_balance=format(account_balance, '.2f'), account_balance=format(account_balance, '.2f'),
ssh_public_keys=ssh_public_keys, ssh_public_keys=ssh_public_keys,
ssh_public_key_count=len(ssh_public_keys), ssh_public_key_count=len(ssh_public_keys),

View File

@ -181,6 +181,7 @@ def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
my_plot.xaxis.set_major_formatter(day_formatter) my_plot.xaxis.set_major_formatter(day_formatter)
max_value = reduce(lambda a, b: a if a > b else b, y, scale) max_value = reduce(lambda a, b: a if a > b else b, y, scale)
average=(sum(y)/len(y))/scale average=(sum(y)/len(y))/scale
average=average*1.25+0.1 average=average*1.25+0.1
@ -195,11 +196,8 @@ def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
my_plot.fill_between( x, y, color=highlight_color, alpha=0.3) my_plot.fill_between( x, y, color=highlight_color, alpha=0.3)
my_plot.plot(x, y, 'r-', color=highlight_color) my_plot.plot(x, y, 'r-', color=highlight_color)
my_plot.patch.set_facecolor('red')
my_plot.patch.set_alpha(0.5)
if size_y < 4: if size_y < 4:
my_plot.set_yticks([0, scale*0.5, scale]) my_plot.set_yticks([0, scale])
my_plot.set_ylim(0, scale) my_plot.set_ylim(0, scale)
my_plot.xaxis.label.set_color(highlight_color) my_plot.xaxis.label.set_color(highlight_color)

View File

@ -71,7 +71,7 @@ CREATE TABLE stripe_checkout_sessions (
INSERT INTO os_images (id, template_image_file_name, description) INSERT INTO os_images (id, template_image_file_name, description)
VALUES ('alpine311', 'alpine-cloud-2020-04-18.qcow2', 'Alpine Linux 3.11'), VALUES ('alpine311', 'alpine-cloud-2020-04-18.qcow2', 'Alpine Linux 3.11'),
('ubuntu18', 'ubuntu-18.04-minimal-cloudimg-amd64.img', 'Ubuntu 18.04 LTS (Bionic Beaver)'), ('ubuntu18', 'ubuntu-18.04-minimal-cloudimg-amd64.img', 'Ubuntu 18.04 LTS (Bionic)'),
('debian10', 'debian-10-genericcloud-amd64-20191117-80.qcow2', 'Debian 10 (Buster)'), ('debian10', 'debian-10-genericcloud-amd64-20191117-80.qcow2', 'Debian 10 (Buster)'),
('centos7', 'CentOS-7-x86_64-GenericCloud.qcow2', 'CentOS 7'), ('centos7', 'CentOS-7-x86_64-GenericCloud.qcow2', 'CentOS 7'),
('centos8', 'CentOS-8-GenericCloud-8.1.1911-20200113.3.x86_64.qcow2', 'CentOS 8'), ('centos8', 'CentOS-8-GenericCloud-8.1.1911-20200113.3.x86_64.qcow2', 'CentOS 8'),
@ -86,6 +86,7 @@ VALUES ('f1-s', 5.33, 512, 1, 500),
('f1-xx', 29.66, 8192, 4, 8000), ('f1-xx', 29.66, 8192, 4, 8000),
('f1-xxx', 57.58, 16384, 8, 16000); ('f1-xxx', 57.58, 16384, 8, 16000);
-- this is test data to be removed later
INSERT INTO accounts (email) INSERT INTO accounts (email)
VALUES ('forest.n.johnson@gmail.com'); VALUES ('forest.n.johnson@gmail.com');

View File

@ -0,0 +1,25 @@
#!/bin/sh
## check avaliable RAM and IPv4s
RAM_BYTES_TO_ALLOCATE="$1"
if echo "$RAM_BYTES_TO_ALLOCATE" | grep -vqE "^[0-9]+$"; then
echo "RAM_BYTES_TO_ALLOCATE \"$RAM_BYTES_TO_ALLOCATE\" must be an integer"
exit 1
fi
#100GB
RAM_LIMIT=100000000000
## compare current allocated ram + RAM_BYTES_TO_ALLOCATE with RAM_LIMIT
IPV4_LIMIT=26
IPV4_COUNT=$(grep ip-add /var/lib/libvirt/dnsmasq/virbr1.status | wc -l)
if [ $IPV4_COUNT -gt $IPV4_LIMIT ]; then
echo "IPv4 address limit reached"
exit 1
fi
echo "yes"

View File

@ -10,7 +10,7 @@ three_dots() {
vmname="$1" vmname="$1"
if echo "$vmname" | grep -vqE '^capsul-[a-z0-9]{10}$'; then if echo "$vmname" | grep -vqE '^(cvm|capsul)-[a-z0-9]{10}$'; then
echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"' echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"'
exit 1 exit 1
fi fi

View File

@ -2,7 +2,7 @@
vmname="$1" vmname="$1"
if echo "$vmname" | grep -vqE '^capsul-[a-z0-9]{10}$'; then if echo "$vmname" | grep -vqE '^(cvm|capsul)-[a-z0-9]{10}$'; then
echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"' echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"'
exit 1 exit 1
fi fi

View File

@ -0,0 +1,3 @@
#!/bin/sh
virsh list --all | grep running | grep -v ' Id' | grep -v -- '----' | awk '{print $2}' | sort

View File

@ -27,10 +27,11 @@
(Must be funded enough to last at least one month at creation time). (Must be funded enough to last at least one month at creation time).
</p> </p>
<p>You must <a href="/console/account-balance">add funds to your account</a> before you can create a Capsul.</p> <p>You must <a href="/console/account-balance">add funds to your account</a> before you can create a Capsul.</p>
{% else %} {% elif no_ssh_public_keys %}
{% if no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p> <p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p> <p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
{% elif not capacity_avaliable %}
<p>Host(s) at capacity. No capsuls can be created at this time. sorry. </p>
{% else %} {% else %}
<pre> <pre>
CAPSUL SIZES CAPSUL SIZES
@ -84,8 +85,6 @@
</div> </div>
{% endif %} {% endif %}
{% endif %}
{% endif %} {% endif %}
</div></div> </div></div>
{% endblock %} {% endblock %}

View File

@ -9,7 +9,7 @@ from subprocess import run
from capsulflask.db import get_model from capsulflask.db import get_model
def validate_capsul_id(id): def validate_capsul_id(id):
if not re.match(r"^capsul-[a-z0-9]{10}$", id): if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"") raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
class VirtualMachine: class VirtualMachine:
@ -19,6 +19,9 @@ class VirtualMachine:
self.ipv6 = ipv6 self.ipv6 = ipv6
class VirtualizationInterface: class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
pass
def get(self, id: str) -> VirtualMachine: def get(self, id: str) -> VirtualMachine:
pass pass
@ -32,6 +35,9 @@ class VirtualizationInterface:
pass pass
class MockVirtualization(VirtualizationInterface): class MockVirtualization(VirtualizationInterface):
def capacity_avaliable(self, additional_ram_bytes):
return True
def get(self, id): def get(self, id):
validate_capsul_id(id) validate_capsul_id(id)
return VirtualMachine(id, ipv4="1.1.1.1") return VirtualMachine(id, ipv4="1.1.1.1")
@ -63,6 +69,27 @@ class ShellScriptVirtualization(VirtualizationInterface):
{completedProcess.stderr} {completedProcess.stderr}
""") """)
def capacity_avaliable(self, additional_ram_bytes):
my_args=[join(current_app.root_path, 'shell_scripts/capacity-avaliable.sh'), additional_ram_bytes]
completedProcess = run(my_args, capture_output=True)
if completedProcess.returncode != 0:
print(f"""
capacity-avaliable.sh exited {completedProcess.returncode} with
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")
return False
lines = completedProcess.stdout.splitlines()
if not lines[len(lines)-1] == "yes":
print("capacity-avaliable.sh exited 0 but did not return \"yes\" ")
return False
return True
def get(self, id): def get(self, id):
validate_capsul_id(id) validate_capsul_id(id)
completedProcess = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True) completedProcess = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True)
@ -75,7 +102,7 @@ class ShellScriptVirtualization(VirtualizationInterface):
return VirtualMachine(id, ipv4=lines[0]) return VirtualMachine(id, ipv4=lines[0])
def list_ids(self) -> list: def list_ids(self) -> list:
completedProcess = run([join(current_app.root_path, 'shell_scripts/list_ids.sh')], capture_output=True) completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True)
self.validate_completed_process(completedProcess) self.validate_completed_process(completedProcess)
return completedProcess.stdout.splitlines() return completedProcess.stdout.splitlines()

View File

@ -14,4 +14,10 @@ packages = find:
include_package_data = true include_package_data = true
install_requires = install_requires =
Flask Flask
Flask-Mail
psycopg2 psycopg2
nanoid
matplotlib
stripe
python-dotenv
requests