better error handling when the btcpay server goes down

* forest's ReadMe docs changes
* add Configuration-type-stuff that lives in the database

.gitignore

@@ -1,5 +1,6 @@
 notes.txt
 .env
+.env.bak
 .vscode
 
 *.pyc

capsulflask/__init__.py

@@ -140,10 +140,14 @@ else:
 app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
 
 app.config['BTCPAY_ENABLED'] = False
-if app.config['BTCPAY_URL'] is not "":
+if app.config['BTCPAY_URL'] != "":
   try:
-    app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
-    app.config['BTCPAY_ENABLED'] = True
+    response = requests.get(app.config['BTCPAY_URL'])
+    if response.status_code == 200:
+      app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
+      app.config['BTCPAY_ENABLED'] = True
+    else:
+      app.logger.warning(f"Can't reach BTCPAY_URL {app.config['BTCPAY_URL']}: Response status code: {response.status_code}. Capsul will work fine except cryptocurrency payments will not work.")
   except:
     app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
 

capsulflask/cli.py

@@ -62,6 +62,18 @@ def sql_script(f, c):
 
     model.connection.commit()
 
+@bp.cli.command('account-balance')
+@click.option('-u', help='users email address')
+@with_appcontext
+def account_balance(u):
+  vms = get_model().list_vms_for_account(u)
+  payments = get_model().list_payments_for_account(u)
+  click.echo(".")
+  click.echo(".")
+  click.echo(get_account_balance(vms, payments, datetime.utcnow()))
+  click.echo(".")
+
+
 
 @bp.cli.command('cron-task')
 @with_appcontext

capsulflask/console.py

@@ -199,6 +199,14 @@ def create():
   capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
   errors = list()
 
+  affordable_vm_sizes = dict()
+  for key, vm_size in vm_sizes.items():
+    # if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month, 
+    # then they have to delete the vm and re-create it, they will not be able to, they will have to pay again.
+    # so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake
+    if vm_size["dollars_per_month"] <= account_balance+0.25:
+      affordable_vm_sizes[key] = vm_size
+
   if request.method == "POST":
     if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
       return abort(418, f"u want tea")
@@ -209,6 +217,8 @@ def create():
       errors.append("Size is required")
     elif size not in vm_sizes:
       errors.append(f"Invalid size {size}")
+    elif size not in affordable_vm_sizes:
+      errors.append(f"Your account must have enough credit to run an {size} for 1 month before you will be allowed to create it")
 
     if not os:
       errors.append("OS is required")
@@ -260,13 +270,6 @@ def create():
       
       return redirect(f"{url_for('console.index')}?created={id}")
   
-  affordable_vm_sizes = dict()
-  for key, vm_size in vm_sizes.items():
-    # if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month, 
-    # then they have to delete the vm and re-create it, they will not be able to, they will have to pay again.
-    # so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake
-    if vm_size["dollars_per_month"] <= account_balance+0.25:
-      affordable_vm_sizes[key] = vm_size
 
   for error in errors:
     flash(error)

capsulflask/db.py

@@ -43,7 +43,7 @@ def init_app(app, is_running_server):
   hasSchemaVersionTable = False
   actionWasTaken = False
   schemaVersion = 0
-  desiredSchemaVersion = 18
+  desiredSchemaVersion = 20
 
   cursor = connection.cursor()
 

capsulflask/hub_model.py

@@ -17,6 +17,10 @@ from capsulflask.http_client import HTTPResult
 from capsulflask.shared import VirtualizationInterface, VirtualMachine, OnlineHost, validate_capsul_id, my_exec_info_message
 
 class MockHub(VirtualizationInterface):
+  def __init__(self):
+    self.default_network = "public1"
+    self.default_ipv4 = "1.1.1.1"
+
   def capacity_avaliable(self, additional_ram_bytes):
     return True
 
@@ -29,9 +33,9 @@ class MockHub(VirtualizationInterface):
         {"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"},
         {"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"}
       ]""")
-      return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", ssh_host_keys=ssh_host_keys)
+      return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4, ssh_host_keys=ssh_host_keys)
 
-    return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1")
+    return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4)
 
   def list_ids(self) -> list:
     return get_model().all_non_deleted_vm_ids()
@@ -40,6 +44,16 @@ class MockHub(VirtualizationInterface):
     validate_capsul_id(id)
     current_app.logger.info(f"mock create: {id} for {email}")
     sleep(1)
+    get_model().create_vm(
+      email=email, 
+      id=id, 
+      size=size, 
+      os=os,
+      host=current_app.config["SPOKE_HOST_ID"],
+      network_name=self.default_network,
+      public_ipv4=self.default_ipv4,
+      ssh_authorized_keys=list(map(lambda x: x["name"], ssh_authorized_keys)),
+    )
 
   def destroy(self, email: str, id: str):
     current_app.logger.info(f"mock destroy: {id} for {email}")
@@ -49,7 +63,6 @@ class MockHub(VirtualizationInterface):
 
 
 class CapsulFlaskHub(VirtualizationInterface):
-
   def synchronous_operation(self, hosts: List[OnlineHost], email: str, payload: str) -> List[HTTPResult]:
     return self.generic_operation(hosts, email, payload, True)[1]
   
@@ -215,8 +228,8 @@ class CapsulFlaskHub(VirtualizationInterface):
         # no need to do anything here since if it cant be parsed then generic_operation will handle it.
         pass
       
+    assigned_hosts_string = ", ".join(assigned_hosts)
     if number_of_assigned != 1:
-      assigned_hosts_string = ", ".join(assigned_hosts)
       raise ValueError(f"expected create capsul operation {operation_id} to be assigned to one host, it was assigned to {number_of_assigned} ({assigned_hosts_string})")
     if error_message != "":
       raise ValueError(f"create capsul operation {operation_id} on {assigned_hosts_string} failed with {error_message}")
@@ -262,4 +275,3 @@ class CapsulFlaskHub(VirtualizationInterface):
 
     if not result_status == "success":
       raise ValueError(f"""failed to {command} vm "{id}" on host "{host.id}" for {email}: {result_json_string}""")
-

capsulflask/payment.py

@@ -58,14 +58,19 @@ def btcpay_payment():
     dollars = result[1]
 
     if len(errors) == 0:
-      invoice = current_app.config['BTCPAY_CLIENT'].create_invoice(dict(
-        price=float(dollars), 
-        currency="USD",
-        itemDesc="Capsul Cloud Compute",
-        transactionSpeed="high",
-        redirectURL=f"{current_app.config['BASE_URL']}/console/account-balance",
-        notificationURL=f"{current_app.config['BASE_URL']}/payment/btcpay/webhook"
-      ))
+      try:
+        invoice = current_app.config['BTCPAY_CLIENT'].create_invoice(dict(
+          price=float(dollars), 
+          currency="USD",
+          itemDesc="Capsul Cloud Compute",
+          transactionSpeed="high",
+          redirectURL=f"{current_app.config['BASE_URL']}/console/account-balance",
+          notificationURL=f"{current_app.config['BASE_URL']}/payment/btcpay/webhook"
+        ))
+      except:
+        current_app.logger.error(f"An error occurred while attempting to reach BTCPay Server: {my_exec_info_message(sys.exc_info())}")
+        flash("An error occurred while attempting to reach BTCPay Server.")
+        return redirect(url_for("console.account_balance"))
 
       current_app.logger.info(f"created btcpay invoice: {invoice}")
 

capsulflask/schema_migrations/19_down_os_updates.sql

@@ -0,0 +1,8 @@
+DELETE FROM os_images WHERE id = 'guixsystem130';
+DELETE FROM os_images WHERE id = 'archlinux';
+UPDATE os_images SET deprecated = FALSE WHERE id = 'guixsystem120';
+UPDATE os_images SET deprecated = FALSE WHERE id = 'centos7';
+UPDATE os_images SET deprecated = FALSE WHERE id = 'centos8';
+UPDATE os_images SET description = 'Ubuntu 20.04 LTS (Fossa)' WHERE id = 'ubuntu20';
+
+UPDATE schemaversion SET version = 18;

capsulflask/schema_migrations/19_up_os_updates.sql

@@ -0,0 +1,12 @@
+INSERT INTO os_images (id, template_image_file_name, description, deprecated)
+VALUES  ('guixsystem130', 'guixsystem/1.3.0/root.img.qcow2', 'Guix System 1.3.0', FALSE);
+INSERT INTO os_images (id, template_image_file_name, description, deprecated)
+VALUES  ('archlinux', 'archlinux/root.img.qcow2', 'Arch Linux', FALSE);
+
+UPDATE os_images SET deprecated = TRUE WHERE id = 'guixsystem120';
+UPDATE os_images SET deprecated = TRUE WHERE id = 'centos7';
+UPDATE os_images SET deprecated = TRUE WHERE id = 'centos8';
+UPDATE os_images SET description = 'Ubuntu 20.04 (Focal)' WHERE id = 'ubuntu20';
+
+
+UPDATE schemaversion SET version = 19;

capsulflask/schema_migrations/20_down_deferrable_constraints.sql

@@ -0,0 +1,21 @@
+
+ALTER TABLE host_network ALTER CONSTRAINT host_network_host_fkey NOT DEFERRABLE;
+ALTER TABLE host_operation ALTER CONSTRAINT host_operation_host_fkey NOT DEFERRABLE;
+ALTER TABLE host_operation ALTER CONSTRAINT host_operation_operation_fkey NOT DEFERRABLE;
+ALTER TABLE login_tokens ALTER CONSTRAINT login_tokens_email_fkey NOT DEFERRABLE;
+ALTER TABLE operations ALTER CONSTRAINT operations_email_fkey NOT DEFERRABLE;
+ALTER TABLE payment_sessions ALTER CONSTRAINT payment_sessions_email_fkey NOT DEFERRABLE;
+ALTER TABLE payments ALTER CONSTRAINT payments_email_fkey NOT DEFERRABLE;
+ALTER TABLE ssh_public_keys ALTER CONSTRAINT ssh_public_keys_email_fkey NOT DEFERRABLE;
+ALTER TABLE unresolved_btcpay_invoices ALTER CONSTRAINT unresolved_btcpay_invoices_email_fkey NOT DEFERRABLE;
+ALTER TABLE unresolved_btcpay_invoices ALTER CONSTRAINT unresolved_btcpay_invoices_email_payment_id_fkey NOT DEFERRABLE;
+ALTER TABLE vm_ssh_authorized_key ALTER CONSTRAINT vm_ssh_public_key_email_ssh_public_key_name_fkey NOT DEFERRABLE;
+ALTER TABLE vm_ssh_authorized_key ALTER CONSTRAINT vm_ssh_public_key_email_vm_id_fkey NOT DEFERRABLE;
+ALTER TABLE vm_ssh_host_key ALTER CONSTRAINT vm_ssh_host_key_email_vm_id_fkey NOT DEFERRABLE;
+ALTER TABLE vms ALTER CONSTRAINT vms_email_fkey NOT DEFERRABLE;
+ALTER TABLE vms ALTER CONSTRAINT vms_host_network_name_fkey NOT DEFERRABLE;
+ALTER TABLE vms ALTER CONSTRAINT vms_host_fkey NOT DEFERRABLE;
+ALTER TABLE vms ALTER CONSTRAINT vms_os_fkey NOT DEFERRABLE;
+ALTER TABLE vms ALTER CONSTRAINT vms_size_fkey NOT DEFERRABLE;
+
+UPDATE schemaversion SET version = 19;

capsulflask/schema_migrations/20_up_deferrable_constraints.sql

@@ -0,0 +1,21 @@
+
+ALTER TABLE host_network ALTER CONSTRAINT host_network_host_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE host_operation ALTER CONSTRAINT host_operation_host_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE host_operation ALTER CONSTRAINT host_operation_operation_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE login_tokens ALTER CONSTRAINT login_tokens_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE operations ALTER CONSTRAINT operations_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE payment_sessions ALTER CONSTRAINT payment_sessions_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE payments ALTER CONSTRAINT payments_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE ssh_public_keys ALTER CONSTRAINT ssh_public_keys_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE unresolved_btcpay_invoices ALTER CONSTRAINT unresolved_btcpay_invoices_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE unresolved_btcpay_invoices ALTER CONSTRAINT unresolved_btcpay_invoices_email_payment_id_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vm_ssh_authorized_key ALTER CONSTRAINT vm_ssh_public_key_email_ssh_public_key_name_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vm_ssh_authorized_key ALTER CONSTRAINT vm_ssh_public_key_email_vm_id_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vm_ssh_host_key ALTER CONSTRAINT vm_ssh_host_key_email_vm_id_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vms ALTER CONSTRAINT vms_email_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vms ALTER CONSTRAINT vms_host_network_name_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vms ALTER CONSTRAINT vms_host_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vms ALTER CONSTRAINT vms_os_fkey DEFERRABLE INITIALLY DEFERRED;
+ALTER TABLE vms ALTER CONSTRAINT vms_size_fkey DEFERRABLE INITIALLY DEFERRED;
+
+UPDATE schemaversion SET version = 20;

capsulflask/spoke_api.py

@@ -182,7 +182,10 @@ def handle_destroy(operation_id, request_body):
     return abort(400, f"bad request; email is required for destroy")
 
   try:
-    current_app.config['SPOKE_MODEL'].destroy(id=request_body['id'], email=request_body['email'])
+    vm = current_app.config['SPOKE_MODEL'].get(request_body['id'], False)
+    current_app.logger.warning(f"destroy {request_body['id']} was called for {request_body['email']}, however the vm does not exist. returning success. ")
+    if vm is not None:
+      current_app.config['SPOKE_MODEL'].destroy(id=request_body['id'], email=request_body['email'])
   except:
     error_message = my_exec_info_message(sys.exc_info())
     params =          f"email='{request_body['email'] if 'email' in request_body else 'KeyError'}', "

capsulflask/templates/support.html

@@ -7,18 +7,11 @@
     <h1>SUPPORT</h1>
   </div>
   <div class="row half-margin">
-    <a href="mailto:support@cyberia.club?subject=Please%20help!">support@cyberia.club</a> 
+    <a href="mailto:support@cyberia.club?subject=capsul%20support%20request">support@cyberia.club</a>
   </div>
 {% endblock %}
 
 {% block subcontent %}
-<p>
-  Note: We maintain a searchable archive of all support emails at 
-  <a href="https://lists.cyberia.club/~cyberia/support">https://lists.cyberia.club/~cyberia/support</a>
-</p>
-<p>
-  If you do not want your mail to appear in a public archive, email <a href="mailto:capsul@cyberia.club?subject=Please%20help!">capsul@cyberia.club</a> instead.
-</p>
 <p>
   Please describe your problem or feature request, and we will do our best to get back to you promptly. Thank you very much.
 </p>

docs/database.md

@@ -48,3 +48,20 @@ $ pipenv run flask cli sql -c "SELECT id, created, email, dollars, invalidated f
 ## how to view the logs on the database server (legion.cyberia.club)
 
 `sudo -u postgres pg_dump capsul-flask | gzip -9 > capsul-backup-2021-02-15.gz`
+
+
+## changing the email address on an account
+
+```
+UPDATE accounts SET lower_case_email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE accounts SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE login_tokens SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE operations SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE payment_sessions SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE payments SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE ssh_public_keys SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE unresolved_btcpay_invoices SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE vm_ssh_authorized_key SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE vm_ssh_host_key SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+UPDATE vms SET email = 'new@email.address' WHERE email = 'old@email.address' ;
+```