autonomic-cooperative/capsul-flask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

short-term and long-term capsuls

Browse Source
This commit is contained in:
forest 2022-02-08 17:16:52 -06:00
parent 2279f17a7f
commit ec0d3b1740
13 changed files with 212 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
capsulflask/cli.py
View File

@ -12,8 +12,8 @@ from psycopg2 import ProgrammingError
from flask_mail import Message
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.console import get_account_balance
from capsulflask.shared import my_exec_info_message, get_account_balance
from capsulflask.payment import check_if_shortterm_flag_can_be_unset
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
bp = Blueprint('cli', __name__)
@ -85,11 +85,16 @@ def cron_task():
clean_up_unresolved_btcpay_invoices()
current_app.logger.info("cron_task: finished clean_up_unresolved_btcpay_invoices")
# notify when funds run out
# notify when funds are about to run out and delete long-term vms once account reaches -$10
current_app.logger.info("cron_task: starting notify_users_about_account_balance")
notify_users_about_account_balance()
current_app.logger.info("cron_task: finished notify_users_about_account_balance")
# delete short-term vms and notify user once account reaches $0
current_app.logger.info("cron_task: starting delete_shortterm_vms_if_account_is_empty")
delete_shortterm_vms_if_account_is_empty()
current_app.logger.info("cron_task: finished delete_shortterm_vms_if_account_is_empty")
# make sure vm system and DB are synced
current_app.logger.info("cron_task: starting ensure_vms_and_db_are_synced")
ensure_vms_and_db_are_synced()
@ -118,7 +123,11 @@ def clean_up_unresolved_btcpay_invoices():
f"resolving btcpay invoice {invoice_id} "
f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as completed "
)
get_model().btcpay_invoice_resolved(invoice_id, True)
resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True)
if resolved_invoice_email is not None:
check_if_shortterm_flag_can_be_unset(resolved_invoice_email)
elif days >= 1:
current_app.logger.info(
f"resolving btcpay invoice {invoice_id} "
@ -234,7 +243,12 @@ def notify_users_about_account_balance():
balance_now = get_account_balance(vms, payments, datetime.utcnow())
current_warning = account['account_balance_warning']
pluralize_capsul = "s" if len(vms) > 1 else ""
longterm_vms = list(filter(lambda vm: vm['shortterm'] == False, vms))
if len(longterm_vms) == 0:
continue
pluralize_capsul = "s" if len(longterm_vms) > 1 else ""
warnings = get_warnings_list()
current_warning_index = -1
@ -262,12 +276,48 @@ def notify_users_about_account_balance():
)
get_model().set_account_balance_warning(account['email'], warnings[index_to_send]['id'])
if index_to_send == len(warnings)-1:
for vm in vms:
for vm in longterm_vms:
current_app.logger.warning(f"cron_task: deleting {vm['id']} ( {account['email']} ) due to negative account balance.")
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
get_model().delete_vm(email=account["email"], id=vm['id'])
def delete_shortterm_vms_if_account_is_empty():
accounts = get_model().all_accounts()
for account in accounts:
vms = get_model().list_vms_for_account(account['email'])
payments = get_model().list_payments_for_account(account['email'])
balance = get_account_balance(vms, payments, datetime.utcnow())
shortterm_vms = list(filter(lambda vm: vm['shortterm'] == True, vms))
if len(shortterm_vms) > 0 and balance <= 0:
pluralize_capsul = "s" if len(shortterm_vms) > 1 else ""
pluralize_past_tense = "have" if len(shortterm_vms) > 1 else "has"
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
f"Short-term Capsul{pluralize_capsul} Deleted",
body=(
f"You have run out of funds! Your Short-term Capsul{pluralize_capsul} {pluralize_past_tense} been deleted.\n\n"
),
sender=current_app.config["MAIL_DEFAULT_SENDER"],
recipients=[account['email']]
)
)
for vm in shortterm_vms:
current_app.logger.warning(f"cron_task: deleting shortterm vm {vm['id']} ( {account['email']} ) due to negative account balance.")
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
get_model().delete_vm(email=account["email"], id=vm['id'])
def ensure_vms_and_db_are_synced():
db_vms_by_id = get_all_vms_from_db()
virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)

45
capsulflask/console.py
View File

@ -17,7 +17,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import account_required
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.shared import my_exec_info_message, get_vm_months_float, get_account_balance
from capsulflask.payment import poll_btcpay_session
from capsulflask import cli
@ -201,13 +201,19 @@ def create():
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
errors = list()
affordable_vm_sizes = dict()
month_funded_vm_sizes = dict()
hour_funded_vm_sizes = dict()
for key, vm_size in vm_sizes.items():
# if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month,
# then they have to delete the vm and re-create it, they will not be able to, they will have to pay again.
# so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake
if vm_size["dollars_per_month"] <= account_balance+0.25:
affordable_vm_sizes[key] = vm_size
month_funded_vm_sizes[key] = vm_size
one_month_in_hours = float(730.5)
two_hours_as_fraction_of_month = float(2)/one_month_in_hours
if float(vm_size["dollars_per_month"])*two_hours_as_fraction_of_month <= account_balance:
hour_funded_vm_sizes[key] = vm_size
if request.method == "POST":
if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
@ -219,8 +225,8 @@ def create():
errors.append("Size is required")
elif size not in vm_sizes:
errors.append(f"Invalid size {size}")
elif size not in affordable_vm_sizes:
errors.append(f"Your account must have enough credit to run an {size} for 1 month before you will be allowed to create it")
elif size not in hour_funded_vm_sizes:
errors.append(f"Your account must have enough credit to run an {size} for 2 hours before you will be allowed to create it")
if not os:
errors.append("OS is required")
@ -264,10 +270,11 @@ def create():
id=id,
os=os,
size=size,
shortterm=(size not in month_funded_vm_sizes),
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys))
ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys))
)
return redirect(f"{url_for('console.index')}?created={id}")
@ -278,7 +285,7 @@ def create():
if not capacity_avaliable:
current_app.logger.warning(f"when capsul capacity is restored, send an email to {session['account']}")
return render_template(
"create-capsul.html",
csrf_token = session["csrf-token"],
@ -288,8 +295,9 @@ def create():
ssh_authorized_key_count=len(public_keys_for_account),
no_ssh_public_keys=len(public_keys_for_account) == 0,
operating_systems=operating_systems,
cant_afford=len(affordable_vm_sizes) == 0,
vm_sizes=affordable_vm_sizes
cant_afford=len(hour_funded_vm_sizes) == 0,
vm_sizes=hour_funded_vm_sizes,
month_funded_vm_sizes=month_funded_vm_sizes
)
@bp.route("/ssh", methods=("GET", "POST"))
@ -363,25 +371,6 @@ def get_payments():
return g.user_payments
average_number_of_days_in_a_month = 30.44
def get_vm_months_float(vm, as_of):
end_datetime = vm["deleted"] if vm["deleted"] else as_of
days = float((end_datetime - vm["created"]).total_seconds())/float(60*60*24)
if days < 1:
days = float(1)
return days / average_number_of_days_in_a_month
def get_account_balance(vms, payments, as_of):
vm_cost_dollars = 0.0
for vm in vms:
vm_months = get_vm_months_float(vm, as_of)
vm_cost_dollars += vm_months * float(vm["dollars_per_month"])
payment_dollars_total = float( sum(map(lambda x: 0 if x["invalidated"] else x["dollars"], payments)) )
return payment_dollars_total - vm_cost_dollars
@bp.route("/account-balance")
@account_required

2
capsulflask/db.py
View File

@ -50,7 +50,7 @@ def init_app(app, is_running_server):
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 22
desiredSchemaVersion = 23
cursor = connection.cursor()

28
capsulflask/db_model.py
View File

@ -140,13 +140,13 @@ class DBModel:
def list_vms_for_account(self, email):
self.cursor.execute("""
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.shortterm, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
FROM vms JOIN vm_sizes on vms.size = vm_sizes.id
WHERE vms.email = %s""",
(email, )
)
return list(map(
lambda x: dict(id=x[0], ipv4=x[1], ipv6=x[2], size=x[3], os=x[4], created=x[5], deleted=x[6], dollars_per_month=x[7]),
lambda x: dict(id=x[0], ipv4=x[1], ipv6=x[2], size=x[3], shortterm=x[4], os=x[5], created=x[6], deleted=x[7], dollars_per_month=x[8]),
self.cursor.fetchall()
))
@ -164,12 +164,12 @@ class DBModel:
)
self.connection.commit()
def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys):
def create_vm(self, email, id, size, shortterm, os, host, network_name, public_ipv4, ssh_authorized_keys):
self.cursor.execute("""
INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4)
VALUES (%s, %s, %s, %s, %s, %s, %s)
INSERT INTO vms (email, id, size, shortterm, os, host, network_name, public_ipv4)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""",
(email, id, size, os, host, network_name, public_ipv4)
(email, id, size, shortterm, os, host, network_name, public_ipv4)
)
for ssh_authorized_key in ssh_authorized_keys:
@ -188,7 +188,7 @@ class DBModel:
def get_vm_detail(self, email, id):
self.cursor.execute("""
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted,
vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
vm_sizes.id, vms.shortterm, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month
FROM vms
JOIN os_images on vms.os = os_images.id
JOIN vm_sizes on vms.size = vm_sizes.id
@ -201,7 +201,8 @@ class DBModel:
vm = dict(
id=row[0], ipv4=row[1], ipv6=row[2], os_description=row[3], created=row[4], deleted=row[5],
size=row[6], dollars_per_month=row[7], vcpus=row[8], memory_mb=row[9], bandwidth_gb_per_month=row[10]
size=row[6], shortterm=row[7], dollars_per_month=row[8], vcpus=row[9], memory_mb=row[10],
bandwidth_gb_per_month=row[11],
)
self.cursor.execute("""
@ -221,6 +222,10 @@ class DBModel:
return vm
def clear_shortterm_flag(self, email):
self.cursor.execute("UPDATE vms SET shortterm = FALSE WHERE email = %s", (email))
self.connection.commit()
# ------ PAYMENTS & ACCOUNT BALANCE ---------
@ -303,7 +308,7 @@ class DBModel:
self.cursor.execute( "DELETE FROM payment_sessions WHERE id = %s AND type = %s", (id, payment_type) )
self.connection.commit()
def btcpay_invoice_resolved(self, id, completed):
def btcpay_invoice_resolved(self, id, completed) -> str:
self.cursor.execute("SELECT email, payment_id FROM unresolved_btcpay_invoices WHERE id = %s ", (id,))
row = self.cursor.fetchone()
if row:
@ -312,6 +317,11 @@ class DBModel:
self.cursor.execute("UPDATE payments SET invalidated = TRUE WHERE email = %s id = %s", (row[0], row[1]))
self.connection.commit()
if completed:
return row[0]
return None
def get_unresolved_btcpay_invoices(self):

1
capsulflask/hub_api.py
View File

@ -179,6 +179,7 @@ def on_create_claimed(payload, host_id):
email=payload['email'],
id=payload['id'],
size=payload['size'],
shortterm=payload['shortterm'],
os=payload['os'],
host=host_id,
network_name=payload['network_name'],

9
capsulflask/hub_model.py
View File

@ -38,16 +38,17 @@ class MockHub(VirtualizationInterface):
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4)
def get_all_by_id(self) -> dict:
by_host_and_network = get_model().non_deleted_vms_by_host_and_network()
by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None)
to_return = dict()
for host in by_host_and_network.values():
for network in host.values():
for vm in network:
vm['state'] = vm['desired_state']
to_return[vm['id']] = vm
return to_return
def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
def create(self, email: str, id: str, os: str, size: str, shortterm: bool, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
current_app.logger.info(f"mock create: {id} for {email}")
sleep(1)
@ -55,6 +56,7 @@ class MockHub(VirtualizationInterface):
email=email,
id=id,
size=size,
shortterm=shortterm,
os=os,
host=current_app.config["SPOKE_HOST_ID"],
network_name=self.default_network,
@ -194,7 +196,7 @@ class CapsulFlaskHub(VirtualizationInterface):
return to_return
def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
def create(self, email: str, id: str, os: str, size: str, shortterm: bool, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list):
validate_capsul_id(id)
online_hosts = get_model().get_online_hosts()
#current_app.logger.debug(f"hub_model.create(): ${len(online_hosts)} hosts")
@ -204,6 +206,7 @@ class CapsulFlaskHub(VirtualizationInterface):
id=id,
os=os,
size=size,
shortterm=shortterm,
template_image_file_name=template_image_file_name,
vcpus=vcpus,
memory_mb=memory_mb,

25
capsulflask/payment.py
View File

@ -6,6 +6,7 @@ import re
import sys
from time import sleep
from datetime import datetime, timedelta
from flask import Blueprint
from flask import make_response
from flask import request
@ -21,7 +22,7 @@ from werkzeug.exceptions import abort
from capsulflask.auth import account_required
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.shared import my_exec_info_message, get_account_balance
bp = Blueprint("payment", __name__, url_prefix="/payment")
@ -118,7 +119,11 @@ def poll_btcpay_session(invoice_id):
current_app.logger.info(f"{success_account} paid ${dollars} successfully (btcpay_invoice_id={invoice_id})")
if invoice['status'] == "complete":
get_model().btcpay_invoice_resolved(invoice_id, True)
resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True)
if resolved_invoice_email is not None:
check_if_shortterm_flag_can_be_unset(resolved_invoice_email)
elif invoice['status'] == "expired" or invoice['status'] == "invalid":
get_model().btcpay_invoice_resolved(invoice_id, False)
get_model().delete_payment_session("btcpay", invoice_id)
@ -141,6 +146,16 @@ def btcpay_webhook():
return result[1], result[0]
# if the user has any shortterm vms (vms which were created at a time when they could not pay for that vm for 1 month)
# then it would be nice if those vms could be "promoted" to long-term if they pay up enough to cover costs for 1 month
def check_if_shortterm_flag_can_be_unset(email: str):
vms = get_model().list_vms_for_account(email)
payments = get_model().list_payments_for_account(email)
average_number_of_days_in_a_month = 30.44
balance = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=average_number_of_days_in_a_month))
if balance > 0:
get_model().clear_shortterm_flag(email)
@bp.route("/stripe", methods=("GET", "POST"))
@account_required
@ -247,7 +262,10 @@ def validate_stripe_checkout_session(stripe_checkout_session_id):
#consume_payment_session deletes the checkout session row and inserts a payment row
# its ok to call consume_payment_session more than once because it only takes an action if the session exists
success_email = get_model().consume_payment_session("stripe", stripe_checkout_session_id, dollars)
if success_email is not None:
check_if_shortterm_flag_can_be_unset(success_email)
if success_email:
return dict(email=success_email, dollars=dollars)
@ -264,6 +282,7 @@ def success():
paid = validate_stripe_checkout_session(stripe_checkout_session_id)
if paid:
current_app.logger.info(f"{paid['email']} paid ${paid['dollars']} successfully (stripe_checkout_session_id={stripe_checkout_session_id})")
return redirect(url_for("console.account_balance"))
else:
sleep(1)

7
capsulflask/schema_migrations/23_down_ephemeral_vms.sql Normal file
View File

@ -0,0 +1,7 @@
ALTER TABLE vms DROP shortterm desired_state;
UPDATE schemaversion SET version = 22;

12
capsulflask/schema_migrations/23_up_ephemeral_vms.sql Normal file
View File

@ -0,0 +1,12 @@
ALTER TABLE vms ADD COLUMN shortterm BOOLEAN NULL;
UPDATE vms SET shortterm = TRUE WHERE shortterm is NULL;
ALTER TABLE vms ALTER COLUMN shortterm SET NOT NULL;
UPDATE schemaversion SET version = 23;

35
capsulflask/shared.py
View File

@ -1,6 +1,8 @@
import re
import traceback
from datetime import datetime
from flask import current_app
from typing import List
@ -58,6 +60,39 @@ def authorized_as_hub(headers):
return auth_header_value == current_app.config["HUB_TOKEN"]
return False
def get_account_balance(vms, payments, as_of):
vm_cost_dollars = 0.0
for vm in vms:
vm_months = get_vm_months_float(vm, as_of)
vm_cost_dollars += vm_months * float(vm["dollars_per_month"])
payment_dollars_total = float( sum(map(lambda x: 0 if x["invalidated"] else x["dollars"], payments)) )
return payment_dollars_total - vm_cost_dollars
average_number_of_days_in_a_month = 30.44
def get_vm_months_float(vm, as_of):
end_datetime = vm["deleted"] if vm["deleted"] else as_of
days = float((end_datetime - vm["created"]).total_seconds())/float(60*60*24)
# y / m / d
date_when_minimum_run_time_was_changed_to_one_hour = datetime(2022, 2, 7)
if vm["created"] > date_when_minimum_run_time_was_changed_to_one_hour:
one_hour_in_days = float(1)/float(24)
if days < one_hour_in_days:
days = one_hour_in_days
else:
if days < 1:
days = float(1)
return days / average_number_of_days_in_a_month
def my_exec_info_message(exec_info):
traceback_result = traceback.format_tb(exec_info[2])
if isinstance(traceback_result, list):

3
capsulflask/spoke_model.py
View File

@ -40,11 +40,12 @@ class MockSpoke(VirtualizationInterface):
return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running")
def get_all_by_id(self) -> dict:
by_host_and_network = get_model().non_deleted_vms_by_host_and_network()
by_host_and_network = get_model().non_deleted_vms_by_host_and_network(current_app.config["SPOKE_HOST_ID"])
to_return = dict()
for host in by_host_and_network.values():
for network in host.values():
for vm in network:
vm['state'] = vm['desired_state']
to_return[vm['id']] = vm
current_app.logger.info(f"MOCK get_all_by_id: {json.dumps(to_return)}")

10
capsulflask/templates/capsul-detail.html
View File

@ -61,6 +61,10 @@
<label class="align" for="size">Capsul Size</label>
<span id="size">{{ vm['size'] }}</span>
</div>
<div class="row justify-start">
<label class="align" for="shortterm">Short-Term</label>
<span id="shortterm">{{ vm['shortterm'] }}</span>
</div>
<div class="row justify-start">
<label class="align" for="vm_state">State</label>
{% if vm['state'] == 'starting' or vm['state'] == 'stopping' %}
@ -99,6 +103,12 @@
<label class="align" for="ssh_username">SSH Username</label>
<span id="ssh_username">cyberian</span>
</div>
<div class="row justify-start">
<!-- spacer to make authorized keys show up on its own line -->
<label class="align" for="spacer">&nbsp;</label>
<span id="spacer">&nbsp;</span>
</div>
<div class="row justify-start">
<label class="align" for="ssh_authorized_keys">SSH Authorized Keys</label>
<a id="ssh_authorized_keys" href="/console/ssh">{{ vm['ssh_authorized_keys'] }}</a>

29
capsulflask/templates/create-capsul.html
View File

@ -20,15 +20,14 @@
f1-x $27.50 3 4096M 25G 4TB
f1-xx $50.00 4 8192M 25G 5TB
* net is calculated as a per-month average
* vms are billed for a minimum of 24 hours upon creation
* all VMs come standard with one public IPv4 address</pre>
* all VMs come standard with one public IPv4 address
* vms are billed for a minimum of 1 hour upon creation</pre>
<pre>
Your <a href="/console/account-balance">account balance</a>: ${{ account_balance }}
</pre>
{% if cant_afford %}
<p>Please <a href="/console/account-balance">fund your account</a> in order to create Capsuls</p>
<p>(At least one month of funding is required)</p>
<p>(At least two hours worth of funding is required)</p>
{% elif no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
@ -41,9 +40,29 @@
<div class="row justify-start">
<label class="align" for="size">Capsul Size</label>
<select id="size" name="size">
{% for size in vm_sizes.keys() %}<option value="{{ size }}">{{ size }}</option>{% endfor %}
{% for size in vm_sizes.keys() %}
{% if size in month_funded_vm_sizes %}
<option value="{{ size }}">{{ size }}</option>
{% else %}
<option value="{{ size }}">{{ size }} (short-term)</option>
{% endif %}
{% endfor %}
</select>
</div>
<div class="row justify-start">
<pre>
<b>NEW!</b> Short-term and Long-term Capsuls:
You may create a capsul even if your account balance won't
fund it for long. It will be marked "short-term" and won't
be backed up.
Short-term capsuls will be deleted as soon as your account
balance reaches zero, while long-term capsuls get a
non-payment grace period before they're are deleted.
</pre>
</div>
<div class="row justify-start">
<label class="align" for="os">Operating System</label>
<select id="os" name="os">