diff --git a/capsulflask/cli.py b/capsulflask/cli.py index cc7af18..9b17c57 100644 --- a/capsulflask/cli.py +++ b/capsulflask/cli.py @@ -12,8 +12,8 @@ from psycopg2 import ProgrammingError from flask_mail import Message from capsulflask.db import get_model -from capsulflask.shared import my_exec_info_message -from capsulflask.console import get_account_balance +from capsulflask.shared import my_exec_info_message, get_account_balance +from capsulflask.payment import check_if_shortterm_flag_can_be_unset from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information bp = Blueprint('cli', __name__) @@ -85,11 +85,16 @@ def cron_task(): clean_up_unresolved_btcpay_invoices() current_app.logger.info("cron_task: finished clean_up_unresolved_btcpay_invoices") - # notify when funds run out + # notify when funds are about to run out and delete long-term vms once account reaches -$10 current_app.logger.info("cron_task: starting notify_users_about_account_balance") notify_users_about_account_balance() current_app.logger.info("cron_task: finished notify_users_about_account_balance") + # delete short-term vms and notify user once account reaches $0 + current_app.logger.info("cron_task: starting delete_shortterm_vms_if_account_is_empty") + delete_shortterm_vms_if_account_is_empty() + current_app.logger.info("cron_task: finished delete_shortterm_vms_if_account_is_empty") + # make sure vm system and DB are synced current_app.logger.info("cron_task: starting ensure_vms_and_db_are_synced") ensure_vms_and_db_are_synced() @@ -118,7 +123,11 @@ def clean_up_unresolved_btcpay_invoices(): f"resolving btcpay invoice {invoice_id} " f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as completed " ) - get_model().btcpay_invoice_resolved(invoice_id, True) + resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True) + + if resolved_invoice_email is not None: + check_if_shortterm_flag_can_be_unset(resolved_invoice_email) + elif days >= 1: current_app.logger.info( f"resolving btcpay invoice {invoice_id} " @@ -234,7 +243,12 @@ def notify_users_about_account_balance(): balance_now = get_account_balance(vms, payments, datetime.utcnow()) current_warning = account['account_balance_warning'] - pluralize_capsul = "s" if len(vms) > 1 else "" + longterm_vms = list(filter(lambda vm: vm['shortterm'] == False, vms)) + + if len(longterm_vms) == 0: + continue + + pluralize_capsul = "s" if len(longterm_vms) > 1 else "" warnings = get_warnings_list() current_warning_index = -1 @@ -262,12 +276,48 @@ def notify_users_about_account_balance(): ) get_model().set_account_balance_warning(account['email'], warnings[index_to_send]['id']) if index_to_send == len(warnings)-1: - for vm in vms: + for vm in longterm_vms: current_app.logger.warning(f"cron_task: deleting {vm['id']} ( {account['email']} ) due to negative account balance.") current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id']) get_model().delete_vm(email=account["email"], id=vm['id']) +def delete_shortterm_vms_if_account_is_empty(): + accounts = get_model().all_accounts() + for account in accounts: + vms = get_model().list_vms_for_account(account['email']) + payments = get_model().list_payments_for_account(account['email']) + balance = get_account_balance(vms, payments, datetime.utcnow()) + shortterm_vms = list(filter(lambda vm: vm['shortterm'] == True, vms)) + + if len(shortterm_vms) > 0 and balance <= 0: + + pluralize_capsul = "s" if len(shortterm_vms) > 1 else "" + pluralize_past_tense = "have" if len(shortterm_vms) > 1 else "has" + + current_app.config["FLASK_MAIL_INSTANCE"].send( + Message( + f"Short-term Capsul{pluralize_capsul} Deleted", + body=( + f"You have run out of funds! Your Short-term Capsul{pluralize_capsul} {pluralize_past_tense} been deleted.\n\n" + ), + sender=current_app.config["MAIL_DEFAULT_SENDER"], + recipients=[account['email']] + ) + ) + + for vm in shortterm_vms: + current_app.logger.warning(f"cron_task: deleting shortterm vm {vm['id']} ( {account['email']} ) due to negative account balance.") + current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id']) + get_model().delete_vm(email=account["email"], id=vm['id']) + + + + + + + + def ensure_vms_and_db_are_synced(): db_vms_by_id = get_all_vms_from_db() virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id) diff --git a/capsulflask/console.py b/capsulflask/console.py index 5bf5bce..5b8f6ef 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -17,7 +17,7 @@ from nanoid import generate from capsulflask.metrics import durations as metric_durations from capsulflask.auth import account_required from capsulflask.db import get_model -from capsulflask.shared import my_exec_info_message +from capsulflask.shared import my_exec_info_message, get_vm_months_float, get_account_balance from capsulflask.payment import poll_btcpay_session from capsulflask import cli @@ -201,13 +201,19 @@ def create(): capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024) errors = list() - affordable_vm_sizes = dict() + month_funded_vm_sizes = dict() + hour_funded_vm_sizes = dict() for key, vm_size in vm_sizes.items(): # if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month, # then they have to delete the vm and re-create it, they will not be able to, they will have to pay again. # so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake if vm_size["dollars_per_month"] <= account_balance+0.25: - affordable_vm_sizes[key] = vm_size + month_funded_vm_sizes[key] = vm_size + + one_month_in_hours = float(730.5) + two_hours_as_fraction_of_month = float(2)/one_month_in_hours + if float(vm_size["dollars_per_month"])*two_hours_as_fraction_of_month <= account_balance: + hour_funded_vm_sizes[key] = vm_size if request.method == "POST": if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']: @@ -219,8 +225,8 @@ def create(): errors.append("Size is required") elif size not in vm_sizes: errors.append(f"Invalid size {size}") - elif size not in affordable_vm_sizes: - errors.append(f"Your account must have enough credit to run an {size} for 1 month before you will be allowed to create it") + elif size not in hour_funded_vm_sizes: + errors.append(f"Your account must have enough credit to run an {size} for 2 hours before you will be allowed to create it") if not os: errors.append("OS is required") @@ -264,10 +270,11 @@ def create(): id=id, os=os, size=size, + shortterm=(size not in month_funded_vm_sizes), template_image_file_name=operating_systems[os]['template_image_file_name'], vcpus=vm_sizes[size]['vcpus'], memory_mb=vm_sizes[size]['memory_mb'], - ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys)) + ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys)) ) return redirect(f"{url_for('console.index')}?created={id}") @@ -278,7 +285,7 @@ def create(): if not capacity_avaliable: current_app.logger.warning(f"when capsul capacity is restored, send an email to {session['account']}") - + return render_template( "create-capsul.html", csrf_token = session["csrf-token"], @@ -288,8 +295,9 @@ def create(): ssh_authorized_key_count=len(public_keys_for_account), no_ssh_public_keys=len(public_keys_for_account) == 0, operating_systems=operating_systems, - cant_afford=len(affordable_vm_sizes) == 0, - vm_sizes=affordable_vm_sizes + cant_afford=len(hour_funded_vm_sizes) == 0, + vm_sizes=hour_funded_vm_sizes, + month_funded_vm_sizes=month_funded_vm_sizes ) @bp.route("/ssh", methods=("GET", "POST")) @@ -363,25 +371,6 @@ def get_payments(): return g.user_payments -average_number_of_days_in_a_month = 30.44 - -def get_vm_months_float(vm, as_of): - end_datetime = vm["deleted"] if vm["deleted"] else as_of - days = float((end_datetime - vm["created"]).total_seconds())/float(60*60*24) - if days < 1: - days = float(1) - return days / average_number_of_days_in_a_month - -def get_account_balance(vms, payments, as_of): - - vm_cost_dollars = 0.0 - for vm in vms: - vm_months = get_vm_months_float(vm, as_of) - vm_cost_dollars += vm_months * float(vm["dollars_per_month"]) - - payment_dollars_total = float( sum(map(lambda x: 0 if x["invalidated"] else x["dollars"], payments)) ) - - return payment_dollars_total - vm_cost_dollars @bp.route("/account-balance") @account_required diff --git a/capsulflask/db.py b/capsulflask/db.py index d74a526..01fe76d 100644 --- a/capsulflask/db.py +++ b/capsulflask/db.py @@ -50,7 +50,7 @@ def init_app(app, is_running_server): hasSchemaVersionTable = False actionWasTaken = False schemaVersion = 0 - desiredSchemaVersion = 22 + desiredSchemaVersion = 23 cursor = connection.cursor() diff --git a/capsulflask/db_model.py b/capsulflask/db_model.py index 0537dbb..826d25e 100644 --- a/capsulflask/db_model.py +++ b/capsulflask/db_model.py @@ -140,13 +140,13 @@ class DBModel: def list_vms_for_account(self, email): self.cursor.execute(""" - SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month + SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.shortterm, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month FROM vms JOIN vm_sizes on vms.size = vm_sizes.id WHERE vms.email = %s""", (email, ) ) return list(map( - lambda x: dict(id=x[0], ipv4=x[1], ipv6=x[2], size=x[3], os=x[4], created=x[5], deleted=x[6], dollars_per_month=x[7]), + lambda x: dict(id=x[0], ipv4=x[1], ipv6=x[2], size=x[3], shortterm=x[4], os=x[5], created=x[6], deleted=x[7], dollars_per_month=x[8]), self.cursor.fetchall() )) @@ -164,12 +164,12 @@ class DBModel: ) self.connection.commit() - def create_vm(self, email, id, size, os, host, network_name, public_ipv4, ssh_authorized_keys): + def create_vm(self, email, id, size, shortterm, os, host, network_name, public_ipv4, ssh_authorized_keys): self.cursor.execute(""" - INSERT INTO vms (email, id, size, os, host, network_name, public_ipv4) - VALUES (%s, %s, %s, %s, %s, %s, %s) + INSERT INTO vms (email, id, size, shortterm, os, host, network_name, public_ipv4) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) """, - (email, id, size, os, host, network_name, public_ipv4) + (email, id, size, shortterm, os, host, network_name, public_ipv4) ) for ssh_authorized_key in ssh_authorized_keys: @@ -188,7 +188,7 @@ class DBModel: def get_vm_detail(self, email, id): self.cursor.execute(""" SELECT vms.id, vms.public_ipv4, vms.public_ipv6, os_images.description, vms.created, vms.deleted, - vm_sizes.id, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month + vm_sizes.id, vms.shortterm, vm_sizes.dollars_per_month, vm_sizes.vcpus, vm_sizes.memory_mb, vm_sizes.bandwidth_gb_per_month FROM vms JOIN os_images on vms.os = os_images.id JOIN vm_sizes on vms.size = vm_sizes.id @@ -201,7 +201,8 @@ class DBModel: vm = dict( id=row[0], ipv4=row[1], ipv6=row[2], os_description=row[3], created=row[4], deleted=row[5], - size=row[6], dollars_per_month=row[7], vcpus=row[8], memory_mb=row[9], bandwidth_gb_per_month=row[10] + size=row[6], shortterm=row[7], dollars_per_month=row[8], vcpus=row[9], memory_mb=row[10], + bandwidth_gb_per_month=row[11], ) self.cursor.execute(""" @@ -221,6 +222,10 @@ class DBModel: return vm + def clear_shortterm_flag(self, email): + self.cursor.execute("UPDATE vms SET shortterm = FALSE WHERE email = %s", (email)) + self.connection.commit() + # ------ PAYMENTS & ACCOUNT BALANCE --------- @@ -303,7 +308,7 @@ class DBModel: self.cursor.execute( "DELETE FROM payment_sessions WHERE id = %s AND type = %s", (id, payment_type) ) self.connection.commit() - def btcpay_invoice_resolved(self, id, completed): + def btcpay_invoice_resolved(self, id, completed) -> str: self.cursor.execute("SELECT email, payment_id FROM unresolved_btcpay_invoices WHERE id = %s ", (id,)) row = self.cursor.fetchone() if row: @@ -312,6 +317,11 @@ class DBModel: self.cursor.execute("UPDATE payments SET invalidated = TRUE WHERE email = %s id = %s", (row[0], row[1])) self.connection.commit() + + if completed: + return row[0] + + return None def get_unresolved_btcpay_invoices(self): diff --git a/capsulflask/hub_api.py b/capsulflask/hub_api.py index c3dc610..0725d30 100644 --- a/capsulflask/hub_api.py +++ b/capsulflask/hub_api.py @@ -179,6 +179,7 @@ def on_create_claimed(payload, host_id): email=payload['email'], id=payload['id'], size=payload['size'], + shortterm=payload['shortterm'], os=payload['os'], host=host_id, network_name=payload['network_name'], diff --git a/capsulflask/hub_model.py b/capsulflask/hub_model.py index b6d34e9..b0fe8d5 100644 --- a/capsulflask/hub_model.py +++ b/capsulflask/hub_model.py @@ -38,16 +38,17 @@ class MockHub(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4) def get_all_by_id(self) -> dict: - by_host_and_network = get_model().non_deleted_vms_by_host_and_network() + by_host_and_network = get_model().non_deleted_vms_by_host_and_network(None) to_return = dict() for host in by_host_and_network.values(): for network in host.values(): for vm in network: + vm['state'] = vm['desired_state'] to_return[vm['id']] = vm return to_return - def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): + def create(self, email: str, id: str, os: str, size: str, shortterm: bool, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) current_app.logger.info(f"mock create: {id} for {email}") sleep(1) @@ -55,6 +56,7 @@ class MockHub(VirtualizationInterface): email=email, id=id, size=size, + shortterm=shortterm, os=os, host=current_app.config["SPOKE_HOST_ID"], network_name=self.default_network, @@ -194,7 +196,7 @@ class CapsulFlaskHub(VirtualizationInterface): return to_return - def create(self, email: str, id: str, os: str, size: str, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): + def create(self, email: str, id: str, os: str, size: str, shortterm: bool, template_image_file_name: str, vcpus: int, memory_mb: int, ssh_authorized_keys: list): validate_capsul_id(id) online_hosts = get_model().get_online_hosts() #current_app.logger.debug(f"hub_model.create(): ${len(online_hosts)} hosts") @@ -204,6 +206,7 @@ class CapsulFlaskHub(VirtualizationInterface): id=id, os=os, size=size, + shortterm=shortterm, template_image_file_name=template_image_file_name, vcpus=vcpus, memory_mb=memory_mb, diff --git a/capsulflask/payment.py b/capsulflask/payment.py index 3977979..f717751 100644 --- a/capsulflask/payment.py +++ b/capsulflask/payment.py @@ -6,6 +6,7 @@ import re import sys from time import sleep +from datetime import datetime, timedelta from flask import Blueprint from flask import make_response from flask import request @@ -21,7 +22,7 @@ from werkzeug.exceptions import abort from capsulflask.auth import account_required from capsulflask.db import get_model -from capsulflask.shared import my_exec_info_message +from capsulflask.shared import my_exec_info_message, get_account_balance bp = Blueprint("payment", __name__, url_prefix="/payment") @@ -118,7 +119,11 @@ def poll_btcpay_session(invoice_id): current_app.logger.info(f"{success_account} paid ${dollars} successfully (btcpay_invoice_id={invoice_id})") if invoice['status'] == "complete": - get_model().btcpay_invoice_resolved(invoice_id, True) + resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True) + + if resolved_invoice_email is not None: + check_if_shortterm_flag_can_be_unset(resolved_invoice_email) + elif invoice['status'] == "expired" or invoice['status'] == "invalid": get_model().btcpay_invoice_resolved(invoice_id, False) get_model().delete_payment_session("btcpay", invoice_id) @@ -141,6 +146,16 @@ def btcpay_webhook(): return result[1], result[0] +# if the user has any shortterm vms (vms which were created at a time when they could not pay for that vm for 1 month) +# then it would be nice if those vms could be "promoted" to long-term if they pay up enough to cover costs for 1 month +def check_if_shortterm_flag_can_be_unset(email: str): + vms = get_model().list_vms_for_account(email) + payments = get_model().list_payments_for_account(email) + average_number_of_days_in_a_month = 30.44 + balance = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=average_number_of_days_in_a_month)) + if balance > 0: + get_model().clear_shortterm_flag(email) + @bp.route("/stripe", methods=("GET", "POST")) @account_required @@ -247,7 +262,10 @@ def validate_stripe_checkout_session(stripe_checkout_session_id): #consume_payment_session deletes the checkout session row and inserts a payment row # its ok to call consume_payment_session more than once because it only takes an action if the session exists success_email = get_model().consume_payment_session("stripe", stripe_checkout_session_id, dollars) - + + if success_email is not None: + check_if_shortterm_flag_can_be_unset(success_email) + if success_email: return dict(email=success_email, dollars=dollars) @@ -264,6 +282,7 @@ def success(): paid = validate_stripe_checkout_session(stripe_checkout_session_id) if paid: current_app.logger.info(f"{paid['email']} paid ${paid['dollars']} successfully (stripe_checkout_session_id={stripe_checkout_session_id})") + return redirect(url_for("console.account_balance")) else: sleep(1) diff --git a/capsulflask/schema_migrations/23_down_ephemeral_vms.sql b/capsulflask/schema_migrations/23_down_ephemeral_vms.sql new file mode 100644 index 0000000..a78623a --- /dev/null +++ b/capsulflask/schema_migrations/23_down_ephemeral_vms.sql @@ -0,0 +1,7 @@ + + + +ALTER TABLE vms DROP shortterm desired_state; + +UPDATE schemaversion SET version = 22; + diff --git a/capsulflask/schema_migrations/23_up_ephemeral_vms.sql b/capsulflask/schema_migrations/23_up_ephemeral_vms.sql new file mode 100644 index 0000000..a3e72ae --- /dev/null +++ b/capsulflask/schema_migrations/23_up_ephemeral_vms.sql @@ -0,0 +1,12 @@ + + + + +ALTER TABLE vms ADD COLUMN shortterm BOOLEAN NULL; + +UPDATE vms SET shortterm = TRUE WHERE shortterm is NULL; + +ALTER TABLE vms ALTER COLUMN shortterm SET NOT NULL; + +UPDATE schemaversion SET version = 23; + diff --git a/capsulflask/shared.py b/capsulflask/shared.py index 77476d3..4e927e0 100644 --- a/capsulflask/shared.py +++ b/capsulflask/shared.py @@ -1,6 +1,8 @@ import re import traceback + +from datetime import datetime from flask import current_app from typing import List @@ -58,6 +60,39 @@ def authorized_as_hub(headers): return auth_header_value == current_app.config["HUB_TOKEN"] return False +def get_account_balance(vms, payments, as_of): + + vm_cost_dollars = 0.0 + for vm in vms: + vm_months = get_vm_months_float(vm, as_of) + vm_cost_dollars += vm_months * float(vm["dollars_per_month"]) + + payment_dollars_total = float( sum(map(lambda x: 0 if x["invalidated"] else x["dollars"], payments)) ) + + return payment_dollars_total - vm_cost_dollars + + +average_number_of_days_in_a_month = 30.44 + +def get_vm_months_float(vm, as_of): + end_datetime = vm["deleted"] if vm["deleted"] else as_of + days = float((end_datetime - vm["created"]).total_seconds())/float(60*60*24) + + # y / m / d + date_when_minimum_run_time_was_changed_to_one_hour = datetime(2022, 2, 7) + + if vm["created"] > date_when_minimum_run_time_was_changed_to_one_hour: + one_hour_in_days = float(1)/float(24) + if days < one_hour_in_days: + days = one_hour_in_days + else: + if days < 1: + days = float(1) + + return days / average_number_of_days_in_a_month + + + def my_exec_info_message(exec_info): traceback_result = traceback.format_tb(exec_info[2]) if isinstance(traceback_result, list): diff --git a/capsulflask/spoke_model.py b/capsulflask/spoke_model.py index cbe82b0..9568467 100644 --- a/capsulflask/spoke_model.py +++ b/capsulflask/spoke_model.py @@ -40,11 +40,12 @@ class MockSpoke(VirtualizationInterface): return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=ipv4, state="running") def get_all_by_id(self) -> dict: - by_host_and_network = get_model().non_deleted_vms_by_host_and_network() + by_host_and_network = get_model().non_deleted_vms_by_host_and_network(current_app.config["SPOKE_HOST_ID"]) to_return = dict() for host in by_host_and_network.values(): for network in host.values(): for vm in network: + vm['state'] = vm['desired_state'] to_return[vm['id']] = vm current_app.logger.info(f"MOCK get_all_by_id: {json.dumps(to_return)}") diff --git a/capsulflask/templates/capsul-detail.html b/capsulflask/templates/capsul-detail.html index 087ff84..2d7600c 100644 --- a/capsulflask/templates/capsul-detail.html +++ b/capsulflask/templates/capsul-detail.html @@ -61,6 +61,10 @@ {{ vm['size'] }} +
+ + {{ vm['shortterm'] }} +
{% if vm['state'] == 'starting' or vm['state'] == 'stopping' %} @@ -99,6 +103,12 @@ cyberian
+ +
+ + +   +
{{ vm['ssh_authorized_keys'] }} diff --git a/capsulflask/templates/create-capsul.html b/capsulflask/templates/create-capsul.html index 0f8de5e..734ebbe 100644 --- a/capsulflask/templates/create-capsul.html +++ b/capsulflask/templates/create-capsul.html @@ -20,15 +20,14 @@ f1-x $27.50 3 4096M 25G 4TB f1-xx $50.00 4 8192M 25G 5TB - * net is calculated as a per-month average - * vms are billed for a minimum of 24 hours upon creation - * all VMs come standard with one public IPv4 address + * all VMs come standard with one public IPv4 address + * vms are billed for a minimum of 1 hour upon creation
     Your account balance: ${{ account_balance }}
   
{% if cant_afford %}

Please fund your account in order to create Capsuls

-

(At least one month of funding is required)

+

(At least two hours worth of funding is required)

{% elif no_ssh_public_keys %}

You don't have any ssh public keys yet.

You must upload one before you can create a Capsul.

@@ -41,9 +40,29 @@
+
+
+  NEW! Short-term and Long-term Capsuls:
+
+  You may create a capsul even if your account balance won't 
+  fund it for long. It will be marked "short-term" and won't 
+  be backed up.
+
+  Short-term capsuls will be deleted as soon as your account 
+  balance reaches zero, while long-term capsuls get a 
+  non-payment grace period before they're are deleted.
+        
+
+