From f999adaf714e0aba335a686d72e26a196c920217 Mon Sep 17 00:00:00 2001 From: 3wordchant <3wordchant@noreply.git.autonomic.zone> Date: Wed, 21 Jul 2021 23:26:10 +0200 Subject: [PATCH 1/9] Add VMs to the database even with HUB_MODE=mock (#6) https://todo.cyberia.club/~cyberia/services/83 Co-authored-by: 3wc <3wc.cyberia@doesthisthing.work> Co-authored-by: forest Reviewed-on: https://git.autonomic.zone/3wordchant/capsul-flask/pulls/6 Co-authored-by: 3wordchant <3wordchant@noreply.git.autonomic.zone> Co-committed-by: 3wordchant <3wordchant@noreply.git.autonomic.zone> --- capsulflask/hub_model.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/capsulflask/hub_model.py b/capsulflask/hub_model.py index ac2a865..375bb9f 100644 --- a/capsulflask/hub_model.py +++ b/capsulflask/hub_model.py @@ -17,6 +17,10 @@ from capsulflask.http_client import HTTPResult from capsulflask.shared import VirtualizationInterface, VirtualMachine, OnlineHost, validate_capsul_id, my_exec_info_message class MockHub(VirtualizationInterface): + def __init__(self): + self.default_network = "public1" + self.default_ipv4 = "1.1.1.1" + def capacity_avaliable(self, additional_ram_bytes): return True @@ -29,9 +33,9 @@ class MockHub(VirtualizationInterface): {"key_type":"RSA", "content":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCvotgzgEP65JUQ8S8OoNKy1uEEPEAcFetSp7QpONe6hj4wPgyFNgVtdoWdNcU19dX3hpdse0G8OlaMUTnNVuRlbIZXuifXQ2jTtCFUA2mmJ5bF+XjGm3TXKMNGh9PN+wEPUeWd14vZL+QPUMev5LmA8cawPiU5+vVMLid93HRBj118aCJFQxLgrdP48VPfKHFRfCR6TIjg1ii3dH4acdJAvlmJ3GFB6ICT42EmBqskz2MPe0rIFxH8YohCBbAbrbWYcptHt4e48h4UdpZdYOhEdv89GrT8BF2C5cbQ5i9qVpI57bXKrj8hPZU5of48UHLSpXG8mbH0YDiOQOfKX/Mt", "sha256":"ghee6KzRnBJhND2kEUZSaouk7CD6o6z2aAc8GPkV+GQ"}, {"key_type":"ECDSA", "content":"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLLgOoATz9R4aS2kk7vWoxX+lshK63t9+5BIHdzZeFE1o+shlcf0Wji8cN/L1+m3bi0uSETZDOAWMP3rHLJj9Hk=", "sha256":"aCYG1aD8cv/TjzJL0bi9jdabMGksdkfa7R8dCGm1yYs"} ]""") - return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1", ssh_host_keys=ssh_host_keys) + return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4, ssh_host_keys=ssh_host_keys) - return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4="1.1.1.1") + return VirtualMachine(id, current_app.config["SPOKE_HOST_ID"], ipv4=self.default_ipv4) def list_ids(self) -> list: return get_model().all_non_deleted_vm_ids() @@ -40,6 +44,16 @@ class MockHub(VirtualizationInterface): validate_capsul_id(id) current_app.logger.info(f"mock create: {id} for {email}") sleep(1) + get_model().create_vm( + email=email, + id=id, + size=size, + os=os, + host=current_app.config["SPOKE_HOST_ID"], + network_name=self.default_network, + public_ipv4=self.default_ipv4, + ssh_authorized_keys=list(map(lambda x: x["name"], ssh_authorized_keys)), + ) def destroy(self, email: str, id: str): current_app.logger.info(f"mock destroy: {id} for {email}") @@ -49,7 +63,6 @@ class MockHub(VirtualizationInterface): class CapsulFlaskHub(VirtualizationInterface): - def synchronous_operation(self, hosts: List[OnlineHost], email: str, payload: str) -> List[HTTPResult]: return self.generic_operation(hosts, email, payload, True)[1] @@ -262,4 +275,3 @@ class CapsulFlaskHub(VirtualizationInterface): if not result_status == "success": raise ValueError(f"""failed to {command} vm "{id}" on host "{host.id}" for {email}: {result_json_string}""") - From 827ca4a50b74d7293fd6ebf7b326e278a8728beb Mon Sep 17 00:00:00 2001 From: 3wc <3wc.cyberia@doesthisthing.work> Date: Wed, 21 Jul 2021 12:19:28 +0200 Subject: [PATCH 2/9] Auto-generate the pricing table from the database --- capsulflask/landing.py | 2 ++ capsulflask/static/style.css | 3 +-- capsulflask/templates/pricing.html | 36 +++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/capsulflask/landing.py b/capsulflask/landing.py index 3c1781e..8161352 100644 --- a/capsulflask/landing.py +++ b/capsulflask/landing.py @@ -12,9 +12,11 @@ def index(): @bp.route("/pricing") def pricing(): + vm_sizes = get_model().vm_sizes_dict() operating_systems = get_model().operating_systems_dict() return render_template( "pricing.html", + vm_sizes=vm_sizes, operating_systems=operating_systems ) diff --git a/capsulflask/static/style.css b/capsulflask/static/style.css index f4a2d32..515f1a6 100644 --- a/capsulflask/static/style.css +++ b/capsulflask/static/style.css @@ -241,7 +241,6 @@ thead { background: #bdc7b812; } td, th { - padding: 0.1em 1em; } table.small td, table.small th { @@ -378,4 +377,4 @@ footer { border: 1px solid rgba(255, 223, 155, 0.8); box-sizing: border-box; position: relative; -} \ No newline at end of file +} diff --git a/capsulflask/templates/pricing.html b/capsulflask/templates/pricing.html index b985e67..5d34117 100644 --- a/capsulflask/templates/pricing.html +++ b/capsulflask/templates/pricing.html @@ -6,21 +6,37 @@

CAPSUL TYPES & PRICING

+
+ + + + + + + + + + + + + {% for vm_size_key, vm_size in vm_sizes.items() %} + + + + + + + + + {% endfor %} + +
typemonthly*cpusmemssdnet
{{ vm_size_key }}${{ vm_size['dollars_per_month'] }}{{ vm_size['vcpus'] }}{{ vm_size['memory_mb'] }}25G{{ vm_size['bandwidth_gb_per_month'] }}
+
-    type     monthly*  cpus  mem     ssd   net*
-    -----    -------   ----  ---     ---   ---
-    f1-xs    $5.00     1     512M    25G   .5TB
-    f1-s     $7.50     1     1024M   25G   1TB
-    f1-m     $12.50    1     2048M   25G   2TB
-    f1-l     $20.00    2     3072M   25G   3TB
-    f1-x     $27.50    3     4096M   25G   4TB
-    f1-xx    $50.00    4     8192M   25G   5TB
-
     * net is calculated as a per-month average
     * vms are billed for a minimum of 24 hours upon creation
     * all VMs come standard with one public IPv4 address
-  
     
     SUPPORTED OPERATING SYSTEMS:
 

From f3ae9aae23534660806aefcec3e7130d16ce5e94 Mon Sep 17 00:00:00 2001
From: forest 
Date: Wed, 21 Jul 2021 11:53:44 -0500
Subject: [PATCH 3/9] remove class="small" from pricing table cuz normal size
 fits fine

---
 capsulflask/templates/pricing.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/capsulflask/templates/pricing.html b/capsulflask/templates/pricing.html
index 5d34117..6888660 100644
--- a/capsulflask/templates/pricing.html
+++ b/capsulflask/templates/pricing.html
@@ -7,7 +7,7 @@
     

CAPSUL TYPES & PRICING

- +
From bca570882e93233fa1d8458fc44884a007e95f5a Mon Sep 17 00:00:00 2001 From: 3wc <3wc.cyberia@doesthisthing.work> Date: Wed, 21 Jul 2021 00:19:38 +0200 Subject: [PATCH 4/9] Add load_config_vars context processor.. ..to allow accessing config variables in the templates. This removes the need for adding config variables manually to template contexts. --- capsulflask/__init__.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/capsulflask/__init__.py b/capsulflask/__init__.py index f85f268..027ad06 100644 --- a/capsulflask/__init__.py +++ b/capsulflask/__init__.py @@ -31,7 +31,6 @@ load_dotenv(find_dotenv()) app = Flask(__name__) app.config.from_mapping( - BASE_URL=os.environ.get("BASE_URL", default="http://localhost:5000"), SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"), HUB_MODE_ENABLED=os.environ.get("HUB_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'], @@ -219,6 +218,9 @@ def override_url_for(): return dict(url_for=url_for_with_cache_bust) +@app.context_processor +def load_config_vars(): + return dict(config=app.config) def url_for_with_cache_bust(endpoint, **values): """ @@ -244,7 +246,3 @@ def url_for_with_cache_bust(endpoint, **values): values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename] return url_for(endpoint, **values) - - - - From 8b0ce0ba719901984da1dc331d409404919563b5 Mon Sep 17 00:00:00 2001 From: 3wc <3wc.cyberia@doesthisthing.work> Date: Wed, 21 Jul 2021 00:20:38 +0200 Subject: [PATCH 5/9] Hide the BTCPay link if BTCPAY_PRIVATE_KEY un-set --- capsulflask/templates/account-balance.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/capsulflask/templates/account-balance.html b/capsulflask/templates/account-balance.html index 512e24c..de3b7b0 100644 --- a/capsulflask/templates/account-balance.html +++ b/capsulflask/templates/account-balance.html @@ -46,7 +46,9 @@ Add funds with Credit/Debit (stripe) + {% if config['BTCPAY_PRIVATE_KEY'] != "" %}
  • Add funds with Bitcoin/Litecoin/Monero (btcpay)
  • + {% endif %}
  • Cash: email treasurer@cyberia.club
  • From bf7487f4f0b15f0938418d904873d1ae2ebea825 Mon Sep 17 00:00:00 2001 From: 3wc <3wc.cyberia@doesthisthing.work> Date: Wed, 21 Jul 2021 00:22:58 +0200 Subject: [PATCH 6/9] Don't load /btcpay if BTCPAY_PRIVATE_KEY un-set --- capsulflask/payment.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/capsulflask/payment.py b/capsulflask/payment.py index 1029f7d..b11c164 100644 --- a/capsulflask/payment.py +++ b/capsulflask/payment.py @@ -48,6 +48,10 @@ def validate_dollars(): def btcpay_payment(): errors = list() + if current_app.config['BTCPAY_PRIVATE_KEY'] == "": + flash("BTCPay is not enabled on this server") + return redirect(url_for("console.account_balance")) + if request.method == "POST": result = validate_dollars() errors = result[0] @@ -289,4 +293,4 @@ def success(): # except stripe.error.SignatureVerificationError: # print("/payment/stripe/webhook returned 400: invalid signature") # abort(400, "invalid signature") - \ No newline at end of file + From be6e72028ca7ecfd76d1b85d717468a4bb06103e Mon Sep 17 00:00:00 2001 From: forest Date: Wed, 21 Jul 2021 12:12:41 -0500 Subject: [PATCH 7/9] define BTCPAY_ENABLED based on URL and btcpay key, pass it explicitly --- capsulflask/__init__.py | 17 ++++++++--------- capsulflask/console.py | 1 + capsulflask/payment.py | 2 +- capsulflask/templates/account-balance.html | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/capsulflask/__init__.py b/capsulflask/__init__.py index 027ad06..7261bf4 100644 --- a/capsulflask/__init__.py +++ b/capsulflask/__init__.py @@ -71,7 +71,7 @@ app.config.from_mapping( #STRIPE_WEBHOOK_SECRET=os.environ.get("STRIPE_WEBHOOK_SECRET", default="") BTCPAY_PRIVATE_KEY=os.environ.get("BTCPAY_PRIVATE_KEY", default="").replace("\\n", "\n"), - BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club") + BTCPAY_URL=os.environ.get("BTCPAY_URL", default="") ) app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL']) @@ -139,10 +139,13 @@ else: app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS'])) -try: - app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY']) -except: - app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info())) +app.config['BTCPAY_ENABLED'] = False +if app.config['BTCPAY_URL'] is not "": + try: + app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY']) + app.config['BTCPAY_ENABLED'] = True + except: + app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info())) # only start the scheduler and attempt to migrate the database if we are running the app. # otherwise we are running a CLI command. @@ -218,10 +221,6 @@ def override_url_for(): return dict(url_for=url_for_with_cache_bust) -@app.context_processor -def load_config_vars(): - return dict(config=app.config) - def url_for_with_cache_bust(endpoint, **values): """ Add a query parameter based on the hash of the file, this acts as a cache bust diff --git a/capsulflask/console.py b/capsulflask/console.py index e97b805..5da18e9 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -423,6 +423,7 @@ def account_balance(): has_vms=len(vms_billed)>0, vms_billed=vms_billed, warning_text=warning_text, + btcpay_enabled=current_app.config["BTCPAY_ENABLED"], payments=list(map( lambda x: dict( dollars=x["dollars"], diff --git a/capsulflask/payment.py b/capsulflask/payment.py index b11c164..3c41cd9 100644 --- a/capsulflask/payment.py +++ b/capsulflask/payment.py @@ -48,7 +48,7 @@ def validate_dollars(): def btcpay_payment(): errors = list() - if current_app.config['BTCPAY_PRIVATE_KEY'] == "": + if not current_app.config['BTCPAY_ENABLED']: flash("BTCPay is not enabled on this server") return redirect(url_for("console.account_balance")) diff --git a/capsulflask/templates/account-balance.html b/capsulflask/templates/account-balance.html index de3b7b0..3d47e70 100644 --- a/capsulflask/templates/account-balance.html +++ b/capsulflask/templates/account-balance.html @@ -46,7 +46,7 @@ Add funds with Credit/Debit (stripe)
    • notice: stripe will load nonfree javascript
    - {% if config['BTCPAY_PRIVATE_KEY'] != "" %} + {% if btcpay_enabled %}
  • Add funds with Bitcoin/Litecoin/Monero (btcpay)
  • {% endif %} From 0fa7fb28b52e6646dd426c3b959767220c678db4 Mon Sep 17 00:00:00 2001 From: 3wc <3wc.cyberia@doesthisthing.work> Date: Tue, 20 Jul 2021 11:34:56 +0200 Subject: [PATCH 8/9] Split README up into separate files, plus: * forest's ReadMe docs changes * add Configuration-type-stuff that lives in the database --- README.md | 254 +++--------------- docs/architecture.md | 30 +++ docs/btcpay.md | 68 +++++ docs/capsul.webp | Bin 0 -> 10230 bytes docs/configuration.md | 89 ++++++ docs/database.md | 50 ++++ docs/deployment.md | 87 ++++++ .../images}/btcpay_sin_pairing.jpg | Bin .../images}/btcpayment_process.drawio | 0 .../images}/btcpayment_process.png | Bin .../images}/generate_btcpay_keys.py | 0 {readme => docs/images}/hub-and-spoke.xml | 0 {readme => docs/images}/hub-and-spoke1.png | Bin {readme => docs/images}/hub-and-spoke2.png | Bin {readme => docs/images}/paired.jpg | Bin docs/local-set-up.md | 68 +++++ 16 files changed, 431 insertions(+), 215 deletions(-) create mode 100644 docs/architecture.md create mode 100644 docs/btcpay.md create mode 100644 docs/capsul.webp create mode 100644 docs/configuration.md create mode 100644 docs/database.md create mode 100644 docs/deployment.md rename {readme => docs/images}/btcpay_sin_pairing.jpg (100%) rename {readme => docs/images}/btcpayment_process.drawio (100%) rename {readme => docs/images}/btcpayment_process.png (100%) rename {readme => docs/images}/generate_btcpay_keys.py (100%) rename {readme => docs/images}/hub-and-spoke.xml (100%) rename {readme => docs/images}/hub-and-spoke1.png (100%) rename {readme => docs/images}/hub-and-spoke2.png (100%) rename {readme => docs/images}/paired.jpg (100%) create mode 100644 docs/local-set-up.md diff --git a/README.md b/README.md index 1442a7d..267a741 100644 --- a/README.md +++ b/README.md @@ -1,227 +1,51 @@ -# capsulflask +# capsul-flask -Python Flask web application for capsul.org +![screenshot of capsul.org home page](./docs/capsul.webp) +Python Flask web application implementing user accounts, payment, and virtual machine management for a smol "virtual machine (vm) as a service" aka "cloud compute" provider. Originally developed by [Cyberia Computer Club](https://cyberia.club) for https://capsul.org -## how to run locally +`capsul-flask` integrates with [Stripe](https://stripe.com/) as a credit card processor, and [BTCPay Server](https://github.com/btcpayserver/btcpayserver-docker) as a cryptocurrency payment processor. -Ensure you have the pre-requisites for the psycopg2 Postgres database adapter package +`capsul-flask` invokes [shell-scripts](./capsulflask/shell_scripts/) to create/manage [libvirt/qemu](https://www.libvirt.org/manpages/virsh.html) vms, and it depends on `dnsmasq` to act as the DHCP server for the vms. + +`capsul-flask` has a ["hub and spoke" architecture](./architecture.md). The "Hub" runs the web application and talks to the Postrges database, while the "Spoke"(s) are responsible for creating/managing virtual machines. In this way, capsul can be scaled to span more than one machine. One instance of the capsul-flask application can run in both hub mode and spoke mode at the same time, however there must only be one instance of the app running in "Hub" mode at any given time. + +## Quickstart (run capsul-flask on your computer in development mode) ``` -sudo apt install python3-dev libpq-dev -pg_config --version -``` +# get an instance of postgres running locally on port 5432 +# (you don't have to use docker, but we thought this might be the easiest for a how-to example) +docker run --rm -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres & -Ensure you have the wonderful `pipenv` python package management and virtual environment cli +# install dependencies +sudo apt install pipenv python3-dev libpq-dev -``` -sudo apt install pipenv -``` - -Create python virtual environment and install packages - -``` -# install deps +# download and run +git clone https://giit.cyberia.club/~forest/capsul-flask +cd capsul-flask pipenv install -``` - -Run an instance of Postgres (I used docker for this, you can use whatever you want, point is its listening on localhost:5432) - -``` -docker run --rm -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres -``` - -Run the app - -``` pipenv run flask run ``` -Run the app in gunicorn: - -``` -pipenv run gunicorn --bind 127.0.0.1:5000 -k gevent --worker-connections 1000 app:app -``` - -Once you log in for the first time, you will want to give yourself some free capsulbux so you can create fake capsuls for testing. - -Note that by default when running locally, the `SPOKE_MODEL` is set to `mock`, meaning that it won't actually try to spawn vms. - -``` -pipenv run flask cli sql -c "INSERT INTO payments (email, dollars) VALUES ('', 20.00)" -``` - -## configuration: - -Create a `.env` file to set up the application configuration: - -``` -nano .env -``` - -You can enter any environment variables referenced in `__init__.py` to this file. - -For example you may enter your SMTP credentials like this: -``` -MAIL_USERNAME=forest@nullhex.com -MAIL_DEFAULT_SENDER=forest@nullhex.com -MAIL_PASSWORD=************** -``` - -## how to view the logs on the database server (legion.cyberia.club) - -`sudo -u postgres pg_dump capsul-flask | gzip -9 > capsul-backup-2021-02-15.gz` - ------ - -## cli - -You can manually mess around with the database like this: - -``` -pipenv run flask cli sql -f test.sql -``` - -``` -pipenv run flask cli sql -c 'SELECT * FROM vms' -``` - -This one selects the vms table with the column name header: - -``` -pipenv run flask cli sql -c "SELECT string_agg(column_name::text, ', ') from information_schema.columns WHERE table_name='vms'; SELECT * from vms" -``` - -How to modify a payment manually, like if you get a chargeback or to fix customer payment issues: - -``` -$ pipenv run flask cli sql -c "SELECT id, created, email, dollars, invalidated from payments" -1, 2020-05-05T00:00:00, forest.n.johnson@gmail.com, 20.00, FALSE - -$ pipenv run flask cli sql -c "UPDATE payments SET invalidated = True WHERE id = 1" -1 rows affected. - -$ pipenv run flask cli sql -c "SELECT id, created, email, dollars, invalidated from payments" -1, 2020-05-05T00:00:00, forest.n.johnson@gmail.com, 20.00, TRUE -``` - - -How you would kick off the scheduled task: - -``` -pipenv run flask cli cron-task -``` - ------ - -## postgres database schema management - -capsulflask has a concept of a schema version. When the application starts, it will query the database for a table named -`schemaversion` that has one row and one column (`version`). If the `version` it finds is not equal to the `desiredSchemaVersion` variable set in `db.py`, it will run migration scripts from the `schema_migrations` folder one by one until the `schemaversion` table shows the correct version. - -For example, the script named `02_up_xyz.sql` should contain code that migrates the database from schema version 1 to schema version 2. Likewise, the script `02_down_xyz.sql` should contain code that migrates from schema version 2 back to schema version 1. - -**IMPORTANT: if you need to make changes to the schema, make a NEW schema version. DO NOT EDIT the existing schema versions.** - -In general, for safety, schema version upgrades should not delete data. Schema version downgrades will simply throw an error and exit for now. - ------ - -## hub-and-spoke architecture - -![](readme/hub-and-spoke1.png) - -This diagram was created with https://app.diagrams.net/. -To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. - -right now I have 2 types of operations, immediate mode and async. - -both types of operations do assignment synchronously. so if the system cant assign the operation to one or more hosts (spokes), -or whatever the operation requires, then it will fail. - -some operations tolerate partial failures, like, `capacity_avaliable` will succeed if at least one spoke succeeds. -for immediate mode requests (like `list`, `capacity_avaliable`, `destroy`), assignment and completion of the operation are the same thing. - -for async ones, they can be assigned without knowing whether or not they succeeded (`create`). - -![](readme/hub-and-spoke2.png) - -This diagram was created with https://app.diagrams.net/. -To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. - -if you issue a create, and it technically could go to any number of hosts, but only one host responds, it will succeed -but if you issue a create and somehow 2 hosts both think they own that task, it will fail and throw a big error. cuz it expects exactly 1 to own the create task - -currently its not set up to do any polling. its not really like a queue at all. It's all immediate for the most part - ------ - -## how to setup btcpay server - -Generate a private key and the accompanying bitpay SIN for the btcpay API client. - - I used this code as an example: https://github.com/bitpay/bitpay-python/blob/master/bitpay/key_utils.py#L6 - -``` -$ pipenv run python ./readme/generate_btcpay_keys.py -``` - -It should output something looking like this: - -``` ------BEGIN EC PRIVATE KEY----- -EXAMPLEIArx/EXAMPLEKH23EXAMPLEsYXEXAMPLE5qdEXAMPLEcFHoAcEXAMPLEK -oUQDQgAEnWs47PT8+ihhzyvXX6/yYMAWWODluRTR2Ix6ZY7Z+MV7v0W1maJzqeqq -NQ+cpBvPDbyrDk9+Uf/sEaRCma094g== ------END EC PRIVATE KEY----- - - -EXAMPLEwzAEXAMPLEEXAMPLEURD7EXAMPLE -``` - -In order to register the key with the btcpay server, you have to first generate a pairing token using the btcpay server interface. -This requires your btcpay server account to have access to the capsul store. Ask Cass about this. - -Navigate to `Manage store: Access Tokens` at: `https://btcpay.cyberia.club/stores//Tokens` - -![](readme/btcpay_sin_pairing.jpg) - - -Finally, send an http request to the btcpay server to complete the pairing: - -``` -curl -H "Content-Type: application/json" https://btcpay.cyberia.club/tokens -d "{'id': 'EXAMPLEwzAEXAMPLEEXAMPLEURD7EXAMPLE', 'pairingCode': 'XXXXXXX'}" -``` - -It should respond with a token: - -``` -{"data":[{"policies":[],"pairingCode":"XXXXXXX","pairingExpiration":1589473817597,"dateCreated":1589472917597,"facade":"merchant","token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","label":"capsulflask"}]} -``` - -And you should see the token in the btcpay server UI: - -![](readme/paired.jpg) - -Now simply set your `BTCPAY_PRIVATE_KEY` variable in `.env` - -NOTE: make sure to use single quotes and replace the new lines with \n. - -``` -BTCPAY_PRIVATE_KEY='-----BEGIN EC PRIVATE KEY-----\nEXAMPLEIArx/EXAMPLEKH23EXAMPLEsYXEXAMPLE5qdEXAMPLEcFHoAcEXAMPLEK\noUQDQgAEnWs47PT8+ihhzyvXX6/yYMAWWODluRTR2Ix6ZY7Z+MV7v0W1maJzqeqq\nNQ+cpBvPDbyrDk9+Uf/sEaRCma094g==\n-----END EC PRIVATE KEY-----' -``` - - ------ - -## testing cryptocurrency payments - -I used litecoin to test cryptocurrency payments, because its the simplest & lowest fee cryptocurrency that BTCPay server supports. You can download the easy-to-use litecoin SPV wallet `electrum-ltc` from [github.com/pooler/electrum-ltc](https://github.com/pooler/electrum-ltc) or [electrum-ltc.org](https://electrum-ltc.org/), set up a wallet, and then either purchase some litecoin from an exchange, or ask Forest for some litecoin to use for testing. - - -## sequence diagram explaining how BTC payment process works - -![btcpayment_process](readme/btcpayment_process.png) - -This diagram was created with https://app.diagrams.net/. -To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. \ No newline at end of file +Interested in learning more? How about a trip to the the `docs/` folder: + +- [**Setting up capsul-flask locally**](./docs/local-set-up.md) + - [Manually](./docs/local-set-up.md#manually) + - [With docker-compose](./docs/local-set-up.md#docker_compose) +- [**Configuring `capsul-flask`**](./docs/configuration.md) + - [Example configuration from capsul.org (production)](./docs/configuration.md#example) + - [Configuration-type-stuff that lives in the database ](./docs/configuration.md#config_that_lives_in_db) + - [Loading variables from files (docker secrets)](./docs/configuration.md#docker_secrets) +- [**`capsul-flask`'s relationship to its Database Server**](./docs/database.md) + - [Database schema management (schema versions)](./docs/database.md#schema_management) + - [Running manual database queries](./docs/database.md#manual_queries) +- [**`capsul-flask`'s hub-and-spoke architecture**](./docs/architecture.md) +- [**Deploying capsul-flask on a server**](./docs/deployment.md) + - [Installing prerequisites for Spoke Mode](./docs/deployment.md#spoke_mode_prerequisites) + - [Deploying capsul-flask manually](./docs/deployment.md#deploy_manually) + - [Deploying capsul-flask with coop-cloud's docker-swarm configuration](./docs/deployment.md#coop_cloud_docker) + - [Deploying capsul-flask with coop-cloud's `abra` deployment tool](./docs/deployment.md#coop_cloud_abra) +- [**Accepting cryptocurrency payments with BTCPay Server**](./docs/btcpay.md) + - [Setting up the BTCPAY_PRIVATE_KEY](./docs/btcpay.md#BTCPAY_PRIVATE_KEY) + - [Testing cryptocurrency payments](./docs/btcpay.md#testing) + - [Sequence diagram explaining how BTC payment process works (how we accept 0-confirmation transactions 😀)](./docs/btcpay.md#0_conf_diagram) diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..9588606 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,30 @@ +# hub-and-spoke architecture + +The "Hub" runs the web application and talks to the Postrges database, while the "Spoke"s are responsible for creating/managing virtual machines. One instance of the capsul-flask application can run in hub mode and spoke mode at the same time. + +The Hub and the Spoke must be configured to communicate securely with each-other over HTTPS. They both have to be able to dial each-other directly. The URLs / auth tokens they use are configured both in the config file (`HUB_URL`, `SPOKE_HOST_ID`, `SPOKE_HOST_TOKEN` and `HUB_TOKEN`) and in the database (the `id`, `https_url`, and `token` columns in the `hosts` table). + +![](images/hub-and-spoke1.png) + +This diagram was created with https://app.diagrams.net/. +To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. + +right now I have 2 types of operations, immediate mode and async. + +both types of operations do assignment synchronously. so if the system cant assign the operation to one or more hosts (spokes), +or whatever the operation requires, then it will fail. + +some operations tolerate partial failures, like, `capacity_avaliable` will succeed if at least one spoke succeeds. +for immediate mode requests (like `list`, `capacity_avaliable`, `destroy`), assignment and completion of the operation are the same thing. + +for async ones, they can be assigned without knowing whether or not they succeeded (`create`). + +![](images/hub-and-spoke2.png) + +This diagram was created with https://app.diagrams.net/. +To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. + +if you issue a create, and it technically could go to any number of hosts, but only one host responds, it will succeed +but if you issue a create and somehow 2 hosts both think they own that task, it will fail and throw a big error. cuz it expects exactly 1 to own the create task + +currently its not set up to do any polling. its not really like a queue at all. It's all immediate for the most part diff --git a/docs/btcpay.md b/docs/btcpay.md new file mode 100644 index 0000000..4db164a --- /dev/null +++ b/docs/btcpay.md @@ -0,0 +1,68 @@ +## Setting up the BTCPAY_PRIVATE_KEY + +Generate a private key and the accompanying bitpay SIN for the btcpay API client. + +I used this code as an example: https://github.com/bitpay/bitpay-python/blob/master/bitpay/key_utils.py#L6 + +``` +$ pipenv run python ./readme/generate_btcpay_keys.py +``` + +It should output something looking like this: + +``` +-----BEGIN EC PRIVATE KEY----- +EXAMPLEIArx/EXAMPLEKH23EXAMPLEsYXEXAMPLE5qdEXAMPLEcFHoAcEXAMPLEK +oUQDQgAEnWs47PT8+ihhzyvXX6/yYMAWWODluRTR2Ix6ZY7Z+MV7v0W1maJzqeqq +NQ+cpBvPDbyrDk9+Uf/sEaRCma094g== +-----END EC PRIVATE KEY----- + + +EXAMPLEwzAEXAMPLEEXAMPLEURD7EXAMPLE +``` + +In order to register the key with the btcpay server, you have to first generate a pairing token using the btcpay server interface. +This requires your btcpay server account to have access to the capsul store. Ask Cass about this. + +Navigate to `Manage store: Access Tokens` at: `https://btcpay.cyberia.club/stores//Tokens` + +![](images/btcpay_sin_pairing.jpg) + + +Finally, send an http request to the btcpay server to complete the pairing: + +``` +curl -H "Content-Type: application/json" https://btcpay.cyberia.club/tokens -d "{'id': 'EXAMPLEwzAEXAMPLEEXAMPLEURD7EXAMPLE', 'pairingCode': 'XXXXXXX'}" +``` + +It should respond with a token: + +``` +{"data":[{"policies":[],"pairingCode":"XXXXXXX","pairingExpiration":1589473817597,"dateCreated":1589472917597,"facade":"merchant","token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","label":"capsulflask"}]} +``` + +And you should see the token in the btcpay server UI: + +![](images/paired.jpg) + +Now simply set your `BTCPAY_PRIVATE_KEY` variable in `.env` + +NOTE: make sure to use single quotes and replace the new lines with \n. + +``` +BTCPAY_PRIVATE_KEY='-----BEGIN EC PRIVATE KEY-----\nEXAMPLEIArx/EXAMPLEKH23EXAMPLEsYXEXAMPLE5qdEXAMPLEcFHoAcEXAMPLEK\noUQDQgAEnWs47PT8+ihhzyvXX6/yYMAWWODluRTR2Ix6ZY7Z+MV7v0W1maJzqeqq\nNQ+cpBvPDbyrDk9+Uf/sEaRCma094g==\n-----END EC PRIVATE KEY-----' +``` + +----- + +## testing cryptocurrency payments + +I used litecoin to test cryptocurrency payments, because its the simplest & lowest fee cryptocurrency that BTCPay server supports. You can download the easy-to-use litecoin SPV wallet `electrum-ltc` from [github.com/pooler/electrum-ltc](https://github.com/pooler/electrum-ltc) or [electrum-ltc.org](https://electrum-ltc.org/), set up a wallet, and then either purchase some litecoin from an exchange, or [ask Forest for some litecoin](https://sequentialread.com/capsul-rollin-onwards-with-a-web-application/#sqr-comment-container) to use for testing. + + +## sequence diagram explaining how BTC payment process works (how we accept 0-confirmation transactions 😀) + +![btcpayment_process](images/btcpayment_process.png) + +This diagram was created with https://app.diagrams.net/. +To edit it, download the diagram file and edit it with the https://app.diagrams.net/ web application, or you may run the application from [source](https://github.com/jgraph/drawio) if you wish. diff --git a/docs/capsul.webp b/docs/capsul.webp new file mode 100644 index 0000000000000000000000000000000000000000..bdaf665cf73e62879a9e1f1b98aad91f9c73e7ec GIT binary patch literal 10230 zcmZ9yV{j$F)-Jqb+qNf8CbqR>+r}glC$nQ46Wg}UiEZ0XCdN1CJ@2jW*6n(Jbak&@ zs9s&&&#G3Dk(9im0RS{5M3pp@xJmc^^*Jg4Szz?^5GG&%fDZKaut0iuT3&NnY8BN~ zmEf0eO+Lx}road8{nS;JSJRi->Ez%1ulPj1-I~8&3w#M*RUeKcB(ynXlQe{%;qRa{ zlF45uuQbz!&p8|Xmd}fyp{2}6pd}xaU!6|`7DBrqy)M75sn_Wb(5;|9@%k70drRqd zv6ml6@NVFXBw%^3=k^WkOXZ8?P3)6>*X8wkEoThHR!H;L`a8sz=;y~9*jmY)!i>-8 zzCym~{{w#hz+Wp8`x;s*=5v#IJ1^S(iO0lkF}fD?i;rx zl6Q``^4YH#K4&JheU#YBa~f`)iKpHp{W;%LB1rr9wzazz6PFKEyYpZ73SX{tm~2hx z2RfV6tgR^t)rYuxEr{S-8ox>vA6VwWYU`w#>U)KVj-^XRWP)6lZV-AY^1svV)0*8~ z1hvzl-NMA-AB$Hzd+>1@1o>nw0L(ydY1)~jh1R4oX?Rl*{iP%mWSywD#PxN~-J!9& zvbdFv0hr-hyW9H-OTcd(;@ax8?kKFQhMtNymv}6{Clt{4Da7tft$xELEQ^yP;s`es zyHa|oqwsE4q1)yoo~ozorbB08=sw|^@GQ+Z@}@HS$m7*wq9i!KXY`#WUsC;LqPm#k z!d2hiJx|*no+Lc6V}|F&N^QOuWsB|)#CcVk9oL;FrBF0IUqbpbt&1C+nh09D(0gsT z?*R;^S8h_ox*X-ptrP2fy^o$M-vNEh<+^Ic(#l2Sm~Z-YDO;3` z#>c4vpt;;ko+O!Cjf6#&+*Nt=Is$S5s5yor7>q*-;KA~HuSrt6a;rKH397Kr-&}f8 zWwaIZN5$%<)5Od3e%nmxL&0+a8$Qfy6w~;v8x^EzYMP_|n{XnZ5>YoOAMyW&<^O{U zh{xI3{|L!jaYa~~Bhub2mrP`HtWncMG^Zxu`*9;iTtX}2+hgF{K0xv``etGOuOitD z088O5oG!+H=KkS(Y(qw2g_Ec4{Lw#gG_|~sOo$-Q!`ZX{ljG*&9pQiePhROdL~>XA zeX9-G`9b~9*gw!6x56h4{q7PpyDgq&wY;`u+m!9Iz!lL6`QKE`HwA}N&@OYKiT)n` z50TNT6NyB_DT)H{iT|_#FPv2r&QEKNkfA zI{i1$RisOU| zEp_0Tp5#5jhHx4ap%@--qsVnooC8qTZMv5j>&caSx17m`f=QkwNE?d8mRxIA9E1%q zs*qx}9{FA*9?th;&QHN*k4X=8DBbvxiuD;4?@3&x$33esBXjq#`iBp<#J%?EzYS^_ zHLzr#Ml&}|5XKLL!cS%~Nb}>74%x13I=unk&Xg>TB49ZL%+38K8v`qV0sD=6#v+F8 zCqos1Q{DN~K*%lI_+h&Yq8IS5p%9i#?EnG*Xvg=ohuDde#->6Yx-5<9?Qv_6v~_ic zHQjB_Qd2C@quR{Fx<&# zGl=Mp;0m>bu6k_jL&Oi}Z*dN}lzu}x#=N2h71XF?&712Adpm zoPl6yRG4P=0e?w%HBT8X+9D{*_ZAd5^fhz41`+c5l+fumZ6kOJsaB%J0MrWF?z{Bj zMhV=86NP6<$7!l@nyEL38|3uNxjQ-mW!W|&s`JFpAAy!jzlRcgB=+PrL1)%T5ffia zZ|9N4t+n~sSb!(EcVx?%Q#_O8HdH01N;F>nF%zs1SuQBv5jo?bZqk8qe2tl}&8I>@ zy^~CXLDwPOmDemw$Z4s9dbQ;|k|GVe%88+5qb_ zsUt8GJ|4IoE34{%12PYSl(f(w=k2>?BUK>8e#m{N_U=%6oqgx~dc~auL)T8B1LY4} zfzJ%vn#8ZV$JGE-I-h*Hh53>(UHTDUt@!sn>r_Zj>LO}!qLC(Ud^@s_Fgl32>6W_Y z-*^VyDrITDplT~E7^;-vY>`IFTPZ zV0GgG1`TM8BVrH&_I6f(jzJ=Hg*+gg=^d`3`ocZsQzfQM@HMSPa0NSscFHqp?UqII zjHlM8`m=70B1okssf;c5gs-1TpOk48&!P951S+*w zux9b4k6*qep=ttKIIG8}hOR|U-?Wcr(B%~?d5HJv%lFI~J`076XDXH9P7`#fSwhaX z7~67w>tQIe%)oGbETDr~Hpon-^&7+hH;uI+<0G&PbeEXramv_lA^mnL14q6ki#FB= zO}H%DMKb2sR&Rtv!Wk8%ZIt@o2i)9(m@n5rwP|?Jm6u>Jr`TbtcS~7kjvTpLCs>|g zG>myw;t@k7w(OrI^lx>(&MI>8NZ@X4L+625PwE%+FCl|f0Wb-jZPQdDs5lI5rhM4kw zVVir}WydHI7SoBlv9-u-cC)MmXJHjfo*$(AE;6(*@+*Tt7phpR9pPSFU9Ina(Gy?_ zA@t@MJ%!1kYtCIQ!^1zFEERWOn+}U0CyG59SVX9wi%7VwP#8 zDD3jDdZy9iH0W<(-*_#vBxlNyga--A zRa)CU+AZF(7h0@fZzPk3D~7tmqLNY>KLF!AexXQy4(+O-p{j|eNToFTI1SZk*;pKb z-U~0~k!CeVb@BOHelWJLE0^u>@$^1+d?y>3YV_yrd_k`zd*heS&e}#jm`eg>_BoSP1%?A5) z;_XJTJmgh4YEVn$$`D<|vF(19WKIda6TVJ<{)Dh^e}`f`_Jn1_!a(3y>WjD{_BOdM z+%?c*`Z-&jjGf@G{TJf4V^-j4wS2W~aYXwUa-)Vfu8XwIvJE`QU6EsQQ_z+4;NvBe zW`X;!org2;xmobX5?g!^cDbLWxXN+{`I`s*uf&l}Kb5;gvX)AxUc8#Xf&;p#i7lJV zvZc@npUg)jf8-(7#&7io#x0C>ETXN;1ZSi9Mx^g-*F#_=&2>7Pk>bRe1Cr>dTYtb%_v7+fKDMB1QW~)|>tcyRP4c!xmfcBv780c#4SjJ$VD~ z88p#K4Fz^5e2=GIm*Gi`hf`W8mLH!a$$XSt_iwPtMfJkD9Hr4+&E>uR(xRd>)*|b5NFX zCkRY(&VbT@&Mo1VLoS>&oLDiFxJ{o}Jsjdq&=1K9G+g5$(L#2_!did*G`gNQ;T{=x z@hG-I8E>0zE!_r!UKii!{vG%h@!4dpx0C(`Ota1ae<@dI^h#sku0{A==6 zm|~k29(CBJn#KqEPh*)kz*a+@lUGf99$n1Z}9BJ>_S&)X%;3&PE)}?|Lx1ofkFyHVXBi-`S$Nr+8bLpe`F&spWTX%B&l5vhw0q zn>>6Z+TLx~J}{7(J8k}B(I3#`$IzM|}4qTJ#Hx(H?uw_fUm|WLFn7#t) zxu&qVm**))RvkN2qSHwR9^NQg?|-`S{ZupdulquFnw|*pA5ztFoL(bEB6i7U1?RLjf5jQh2F zYlxg8{^pd;2!{$1nbPtn|EVet-q!Rl*I82|g%aoY0@orooQO89pXbxhcA+ct$SzIq z(5SJ;)HFk@(QJUaE=VrEko4UQJvK_8fxpzXK4#@5-(QjyP4V%=yA>NfIez}K)qlC- zpEo`m#~?!t$p2{+61I>26@xgPw7duL{eFkRlY|3(uD#LjR97wgC4L(iih|d-*UKDBs z8>3x+XM1#m)M+Rpj{24QRdZ*4oxoH^)Zr{m{8Q;77E^7VXT;@lL_psdcvsD*NVNsA zoOr;8o3vL{Y}fml^C3Q0<@3&WotMsE2uV+l|D>kjOB#X{>mj4I5azhZnl5df+E8%@ zYBzRL*Kl#?b{NOIou;hGnj$F%Y;3FnKUFYp-Y;M3u zu>yH0JC~&41aq}&`m|{Mdl9_#24G~Qb8Nwwu|tUv=v^C?pe>>z$uEL}Ul z=$Lu1Ic_N|f0O~yct;RZC@sGPhS4FU{VNDy=Id0%;bjMl5pQuWB2@>|52cKL%0MZs zw-2Oq>t2;KHp90_xPUQKT5I?zn{P=|^f_W@jk4MT_zJfU!u)5Ni&I(PzZutA)EBY* zaT25}>l(QLtir-z1P=QTR(8F|Nd;`&mThn@!?h7zNr!5rvJ?rxcZDWRjMl_8 zhS#~6HE>nOesJ7TFVkLSj?<8^I8Azk6HVWUT3R}d8DDn%A%w3GxMceu`|pQFg{tL} z%J;Lpm<(4Y&)qI#K)R& zGC{{~Pah?(U6_ugK>Q=9Vc8tW=reBBrl0v>;CABz4-sG{n7^!G!TRZA{0C7t((VlQ z)zek7TeAczbKzKKy|aQvKTR`69)2Gc|J-KqHoZT@^DB(?Z2xk22O}FFjzQz8<_+Vk zcj#+ytyNRRv7U&qmSth<@j0c3wRb;reWhJb9PnOvHQTgb!TUR-d7lZBEB?qqOhRp4 z&;aNXbgpl<#3r@1(J7Ms0d4T1tN-KW=NCkeF%M&N+X7CnGb%TxthV6NW8O!gTcj%} zB1fZRsjhqC1J6UHL!!NBNU=iyT$_q7e67C_N&)ceJ7z&)IgvxC#I{U~I|M%1nXTaR zg6y^&T;=UAhCJrmid{)T`5GqoN^kj6?;s>L7>npc zIea~mpT-O;?>CGS47a81rXNP2K5(QI>*GgSyyTKMr*w*Po7n3<`guQ12d<8KnQ41l z?O+juf+vQ`i16JmUCwW_vzk_YE@-g6qrKIKN&}2CpTOg*vn~Ty80Zzh+wN1s5}VD^&z7sxVMWy|QW@Tr6SKt}f%%awH$0m=GXD!ivW){Arg0 zCugQR);J|zt`A*xhfp;8d*NB*&|9LXh}85;OCPO&gFd!wo3r7H$7e}+PI>XoN$^ur zvhzvb=3ra=TS;?A9=4$gi$-bTmC-TUR7OnE00pJ?K zh(uqQAdnM%G~t3Q|?c!FyzrV+7ofZ_1&->uH%sa>Ez90$6$twg_FL z1k1HGXPJAPeExG1XlXqLvW$6ey?VGT-9j18`wR=>`z9*wlV+z&HnAAvYA5WUPSx?C z7EYqF`UI7|$-S{#Kyu9P?lz7Q(C)@J5HAG4f3+7(nJ= zkq$YlDuyi>Kr;8>iFg&leYaxgKrmLksSA1fjE7ARvnkph=#qyxc{NEz&UQ7=8Mzqk ztJ+D1-kxrza~J4YfR0c0@UJ##W-n_Lm-)WF&qC1yt;2?j>?0B;Lg50$n)BIn z!Fpl;NoI7aszgw)s7Tt(j2vCpBjzr z!SL~i#y=Z8x;nugU=|TqP%>7e7v!5EE$T9KI5qTmx!?*o2T(<67k9+UksoKTNBL9no00kYslP7S-D@L8o0ULS;xw?m)!lDVc6~GZ@d>SKg!w-=%)r%z3hu+ zZmrHOnSbBi5$dp*<`N@)4r7oIm?_y_-elnI=B7m+Q_=?0SR{U|d*zokz9v27%gDZNyr)Y#1=9l>p7jG;4+xgv;*5dB_q4uaT(4J| zcY6=k*E?ifMu7>L&oCK75ZP!kf!xL@@EDr9EkB*WpL+H#z7YDT#)K@E8y zYJVYK^pE05JP>~KR^S;$H)#X`T&1I<4P7iO%zl(XxMQN~EYcfc+nE4CkN|8i7zZYg zRY4k}9*hQaKcJ2HWmZhUz?dHi_77SPnRKHQu?5e551RJMYX&c)iUgM9xE4PG&z7)q zjugr#PGlx$w)u_f5%XU0`gw?aC-5np6t5oMtyrdQ>}I5iLlmA?LtG#I^=ZAMXvfgc zF;2+RTG0M38^XXkBZpWi9Sy4avaU|)urf{@+YyykX|7Zeg{Lt+$1R<{X%NTjLYYZ- zwXkFwD}~I}*J~!Ydg)>pf~f<1ZZ%5OSTj@a-uC{h50O%LpXF$s8AeUw+}X!do)M9^ zGUHZ^B&9CcqV*hu)!)K-3i;J=Yvr-1_tBUT*9}>!bKvlJEENqwqLI32E?;m;gwqJ$ zECyz$anD43#Z$@2PbVMC;T=e}NxDb2+AtK>A3CTY5@zFwdkK8n`7+ZttW{7qp0`;6 zx&gODk>MJO(9aErh3L;J{(2af#lH_o+J+w)Q`HXNV5bVD7@Rh*m^co2vD$UpML*w| zP72wWPyH^nT`DPpv5-dYL>uh#?ShQn*V0*EWMqnkvKz6|S3xZW3O=LgSQZgx8qKKk zV!~`M=8Gi7fjr8L;@llAD<>S1m3gN5QYRXIS3^(Isk3X6l~`7u0$mEWS<|#M`o%F4 zL98c=HAS{s^}k&t%pbS$;a~&#KTq7s#VX2{6&P3S1#0;OkQhe2Zj@Nx62Hl%Y_v3S zSc_ANMvcd6K?dpyL^TIul{m5dLTvU~V~H&i9;`B*a37D`W`Fxex)c_Dvd9oRBte5G zZInJnT~>K3XKINkf5`9jN%S$+1}3V3AJ-*{+gJXiT`aAJ+@wp&rP6T}@!8;hy#x5x z@}mJe8^*5>r`u`V(+F%3BZb}R##Z@o(xi0bmE!3BUHCvA5vhWJsuzk~t-mf|Kqb>R za%L&ILe{VV^XH>bw3kRVRGHWy3(_rLE2&yj|K37sgSJYliFO1D^P+Lyt`{U67t8Q$ z^z;v)^MCm)Inf|=62BN=8>2`?F*pqaN9*5$lr4utI;dc>h6LCfw);>@0L8hA^tJx- z^9nNRYL>U1j;e8z9cHc!3p|obQXu+ypytnw@0j?CD7dwwB`rL^SK+?KDQh6R}hY1!D*WHe0JjAt{%bs0t_y0JZU&7SZ^cF_?G%4qjXl z@hFJl%as6>v#07|$S&4LiRai*__raq2sFA}IrPI}3Dd`r_2#QgI>B<8HE_`UyJh?( zG>ojR(3LJH3w(h}Vzxc1wNNNT{Exx1iZpXGF}~(HA;+$zjNb^Z1RngX+}dRUi$V4nH~~**k-P}WxJtqBkZefq_+GM=x8zNvziEXX!=-w(s=*cnFr}v(}%L* zUg|j6>1Pw1BgArUHPg0vyn~R&9XyK-HRV1yKhyK7O6%FLL@fsXm%l!e`%)25u{hY| zcbIyyjl>`>{M}f>$MH!s(7V=pOjG(hMmkUG=7a?IX)4T12e_TTcAeRplBHBEG7dvJ zp-yZP+jvW;Ma(KldF}lQfFxj=pT1CHXHb0H<&aWaB-sv7F9B38O%ozIsHy;3J<}{ttxh^ZO46+ zR}WbOpi=?&twu!ef}J`9MxXdU7rS}GM{M@DBXHF3eN&$Gk6n?9Ha`v}DZ>1EQoH#XJ zu-*B<$`69`X$to5rTNi}9pTs6*C196jF0#nimH{FnQ|D1+5ey}Cqtnc{+ackk1K%x E4{OrM4gdfE literal 0 HcmV?d00001 diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..51ff7ee --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,89 @@ +# Configuring Capsul-Flask + +Create a `.env` file to set up the application configuration: + +``` +nano .env +``` + +You can enter any environment variables referenced in [`__init__.py`](../capsulflask/__init__.py) to this file. + +For example you may enter your SMTP credentials like this: +``` +MAIL_USERNAME=forest@nullhex.com +MAIL_DEFAULT_SENDER=forest@nullhex.com +MAIL_PASSWORD=************** +``` + +## Example configuration from capsul.org (production): + +``` +#LOG_LEVEL=DEBUG + +BASE_URL="https://capsul.org" + +# hub url is used by the SPOKE_MODE to contact the hub. Since this server is the hub, +# this is fine. In fact it runs into problems (routing related?) when I set it to capsul.org. +# similarly the baikal "spoke" (set up in the hosts table in the db) has "http://localhost:5000" as the https_url +HUB_URL="http://localhost:5000" + +HUB_MODE_ENABLED="t" +SPOKE_MODE_ENABLED="t" +HUB_MODEL="capsul-flask" +SPOKE_MODEL="shell-scripts" +SPOKE_HOST_ID="baikal" +SPOKE_HOST_TOKEN="" +HUB_TOKEN="" + +# smtp.. see https://flask-mail.readthedocs.io/en/latest/#configuring-flask-mail +MAIL_SERVER="smtp.nullhex.com" + +# MAIL_USE_SSL means SMTP with STARTTLS +MAIL_USE_SSL=true + +# MAIL_USE_TLS means SMTP wrapped in TLS +MAIL_USE_TLS=false + +MAIL_PORT="465" +MAIL_USERNAME="capsul@nullhex.com" +MAIL_PASSWORD="" +MAIL_DEFAULT_SENDER="capsul@nullhex.com" + +# stripe +STRIPE_SECRET_KEY="sk_live_" +STRIPE_PUBLISHABLE_KEY="pk_live_tGDHY7kBwqC71b4F0N7LZdGl00GZOw0iNJ" + +# internal +SECRET_KEY="" +POSTGRES_CONNECTION_PARAMETERS="sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=postgres.cyberia.club port=5432 ..." + + +# btcpay server +BTCPAY_URL="https://beeteeceepae2.cyberia.club" +BTCPAY_PRIVATE_KEY='-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----' +``` + +## Configuration-type-stuff that lives in the database + + - `hosts` table: + - `id` (corresponds to `SPOKE_HOST_ID` in the config) + - `https_url` + - `token` (corresponds to `SPOKE_HOST_TOKEN` in the config) + - `os_images` table: + - `id` + - `template_image_file_name` + - `description` + - `deprecated` + - `vm_sizes` table: + - `id` + - `dollars_per_month` + - `memory_mb` + - `vcpus` + - `bandwidth_gb_per_month` + +## Loading variables from files (docker secrets) + +To support [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/), you can also load secret values from files – for example, to load `MAIL_PASSWORD` from `/run/secrets/mail_password`, set +```sh +MAIL_PASSWORD_FILE=/run/secrets/mail_password +``` diff --git a/docs/database.md b/docs/database.md new file mode 100644 index 0000000..f192783 --- /dev/null +++ b/docs/database.md @@ -0,0 +1,50 @@ +# capsul-flask's relationship to its Database Server + +Capsul has a ["hub and spoke" architecture](./architecture.md). The "Hub" runs the web application and talks to the Postrges database, while the "Spoke"s are responsible for creating/managing virtual machines. One instance of the capsul-flask application can run in both hub mode and spoke mode at the same time, however there must only be one instance of the app running in "Hub" mode at any given time. + +The Postgres connections parameters are [configurable](./configuration.md). + +## Database schema management (schema versions) + +capsul-flask has a concept of a schema version. When the application starts, it will query the database for a table named `schemaversion` that has one row and one column (`version`). If the `version` it finds is not equal to the `desiredSchemaVersion` variable set in `db.py`, it will run migration scripts from the `schema_migrations` folder one by one until the `schemaversion` table shows the correct version. + +For example, the script named `02_up_xyz.sql` should contain code that migrates the database from schema version 1 to schema version 2. Likewise, the script `02_down_xyz.sql` should contain code that migrates from schema version 2 back to schema version 1. + +**IMPORTANT: if you need to make changes to the schema, make a NEW schema version. DO NOT EDIT the existing schema versions.** + +In general, for safety, schema version upgrades should not delete data. Schema version downgrades will simply throw an error and exit for now. + +## Running manual database queries + +You can manually mess around with the database like this: + +``` +pipenv run flask cli sql -f test.sql +``` + +``` +pipenv run flask cli sql -c 'SELECT * FROM vms' +``` + +This one selects the vms table with the column name header: + +``` +pipenv run flask cli sql -c "SELECT string_agg(column_name::text, ', ') from information_schema.columns WHERE table_name='vms'; SELECT * from vms" +``` + +How to modify a payment manually, like if you get a chargeback or to fix customer payment issues: + +``` +$ pipenv run flask cli sql -c "SELECT id, created, email, dollars, invalidated from payments" +1, 2020-05-05T00:00:00, forest.n.johnson@gmail.com, 20.00, FALSE + +$ pipenv run flask cli sql -c "UPDATE payments SET invalidated = True WHERE id = 1" +1 rows affected. + +$ pipenv run flask cli sql -c "SELECT id, created, email, dollars, invalidated from payments" +1, 2020-05-05T00:00:00, forest.n.johnson@gmail.com, 20.00, TRUE +``` + +## how to view the logs on the database server (legion.cyberia.club) + +`sudo -u postgres pg_dump capsul-flask | gzip -9 > capsul-backup-2021-02-15.gz` diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..e564d78 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,87 @@ +# Deploying Capsul on a server + +Capsul has a ["hub and spoke" architecture](./architecture.md). The "Hub" runs the web application and talks to the Postrges database, while the "Spoke"s are responsible for creating/managing virtual machines. One instance of the capsul-flask application can run in both hub mode and spoke mode at the same time, however there must only be one instance of the app running in "Hub" mode at any given time. + +## Installing prerequisites for Spoke Mode + +On your spoke (see [Architecture](./architecture.md) You'll need `libvirtd`, `dnsmasq`, and `qemu-kvm`, plus a `/tank` diectory with some operating system images in it: + +``` +sudo apt install libvirt-daemon-system virtinst git dnsmasq qemu qemu-kvm +sudo mkdir -p /var/www /tank/{vm,img,config} +sudo mkdir -p /tank/img/debian/10 +cd !$ +sudo wget https://cloud.debian.org/images/cloud/buster/20201023-432/debian-10-genericcloud-amd64-20201023-432.qcow2 -O root.img.qcow2 +``` + +TODO: network set-up +TODO: cyberia-cloudinit.yml + +## Deploying capsul-flask + +### Manually + +Follow the [local set-up instructions](./local-set-up.md) on your server. + +Make sure to set `BASE_URL` correctly, generate your own secret tokens, and +configure your own daemon management for the capsul-flask server (e.g. writing +init scripts, or SystemD unit files). + +Use the suggested `gunicorn` command (with appropriately-set address and port), +instead of `flask run`, to launch the server. + +For example, here is the SystemD service unit file we use in production for `capsul.org`: + +``` +[Unit] +Description=capsul-flask virtual machines as a service +After=network.target + +[Service] +ExecStart=/usr/local/bin/pipenv run gunicorn --bind 127.0.0.1:5000 -k gevent --worker-connections 1000 app:app +Restart=on-failure +WorkingDirectory=/opt/capsul-flask + +[Install] +WantedBy=multi-user.target + +``` + +TODO: cron runner is required to run maintenance tasks for now, but in the future we want to build this into the python based task scheduler. + +### Using Co-op Cloud's vanilla Docker Swarm configuration + +Download the Co-op Cloud swarm `compose.yml`: + +```sh +wget https://git.autonomic.zone/coop-cloud/capsul/src/branch/main/compose.yml +``` + +Optionally, download add-on compose files for Stripe, BTCPay, and Spoke Mode: + +```sh +wget https://git.autonomic.zone/coop-cloud/capsul/src/branch/main/compose.{stripe,btcpay,spoke}.yml +``` + +Then, create a `.env` file and configure appropriately -- you probably want to +define most settings in [the Co-op Cloud `.envrc.sample` +file](https://git.autonomic.zone/coop-cloud/capsul/src/branch/main/.envrc.sample). + +Load the environment variables (using Python `direnv`, or a manual `set -a && source .env && set +a`), insert any necessary secrets, then run the deployment: + +```sh +docker stack deploy -c compose.yml -c compose.stripe.yml your_capsul +``` + +(where you'd add an extra `-c compose.btcpay.yml` for each optional compose file +you want, and set `your_capsul` to the "stack name" you want). + +TODO: cron runner + +### Using Co-op Cloud's `abra` deployment tool + +Follow [the guide in the README for the Co-op Cloud capsul package](https://git.autonomic.zone/coop-cloud/capsul/). + +### Using docker-compose + +TODO diff --git a/readme/btcpay_sin_pairing.jpg b/docs/images/btcpay_sin_pairing.jpg similarity index 100% rename from readme/btcpay_sin_pairing.jpg rename to docs/images/btcpay_sin_pairing.jpg diff --git a/readme/btcpayment_process.drawio b/docs/images/btcpayment_process.drawio similarity index 100% rename from readme/btcpayment_process.drawio rename to docs/images/btcpayment_process.drawio diff --git a/readme/btcpayment_process.png b/docs/images/btcpayment_process.png similarity index 100% rename from readme/btcpayment_process.png rename to docs/images/btcpayment_process.png diff --git a/readme/generate_btcpay_keys.py b/docs/images/generate_btcpay_keys.py similarity index 100% rename from readme/generate_btcpay_keys.py rename to docs/images/generate_btcpay_keys.py diff --git a/readme/hub-and-spoke.xml b/docs/images/hub-and-spoke.xml similarity index 100% rename from readme/hub-and-spoke.xml rename to docs/images/hub-and-spoke.xml diff --git a/readme/hub-and-spoke1.png b/docs/images/hub-and-spoke1.png similarity index 100% rename from readme/hub-and-spoke1.png rename to docs/images/hub-and-spoke1.png diff --git a/readme/hub-and-spoke2.png b/docs/images/hub-and-spoke2.png similarity index 100% rename from readme/hub-and-spoke2.png rename to docs/images/hub-and-spoke2.png diff --git a/readme/paired.jpg b/docs/images/paired.jpg similarity index 100% rename from readme/paired.jpg rename to docs/images/paired.jpg diff --git a/docs/local-set-up.md b/docs/local-set-up.md new file mode 100644 index 0000000..4ed83b4 --- /dev/null +++ b/docs/local-set-up.md @@ -0,0 +1,68 @@ +# How to run Capsul locally + +## Manually + +Ensure you have the pre-requisites for the psycopg2 Postgres database adapter package: + +```sh +sudo apt install python3-dev libpq-dev +pg_config --version +``` + +Ensure you have the wonderful `pipenv` python package management and virtual environment cli: + +```sh +sudo apt install pipenv +``` + +Create python virtual environment and install packages: + +```sh +pipenv install +``` + +Run an instance of Postgres (I used docker for this, you can use whatever you want, point is its listening on `localhost:5432`): + +```sh +docker run --rm -it -e POSTGRES_PASSWORD=dev -p 5432:5432 postgres +``` + +Run the app + +```sh +pipenv run flask run +``` + +or, using Gunicorn: + +```sh +pipenv run gunicorn --bind 127.0.0.1:5000 -k gevent --worker-connections 1000 app:app +``` + +Note that by default when running locally, the `SPOKE_MODEL` is set to `mock`, meaning that it won't actually try to spawn vms. + +## Crediting your account + +Once you log in for the first time, you will want to give yourself some free capsulbux so you can create fake capsuls for testing. + +```sh +pipenv run flask cli sql -c "INSERT INTO payments (email, dollars) VALUES ('', 20.00)" +``` + +## Running scheduled tasks: + +```sh +pipenv run flask cli cron-task +``` + +## Run locally with docker-compose + +If you have Docker and Docker-Compose installed, you can use the +`3wordchant/capsul-flask` Docker image to launch capsul-flask, and a Postgres +database server, for you: + +```sh +docker-compose up +``` + +`capsul-flask` will read settings from your `.env` file as usual; you can set any of the options mentioned in the [configuration documentation](./configuration.md). From 08eb38dc5713419330b21fc3b53c01d2d4179941 Mon Sep 17 00:00:00 2001 From: forest Date: Wed, 21 Jul 2021 16:45:53 -0500 Subject: [PATCH 9/9] correctly enforce affordable_vm_sizes post form submission --- .gitignore | 1 + capsulflask/console.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index bb022e9..c9bb7c2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ notes.txt .env +.env.bak .vscode *.pyc diff --git a/capsulflask/console.py b/capsulflask/console.py index 5da18e9..d15155d 100644 --- a/capsulflask/console.py +++ b/capsulflask/console.py @@ -199,6 +199,14 @@ def create(): capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024) errors = list() + affordable_vm_sizes = dict() + for key, vm_size in vm_sizes.items(): + # if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month, + # then they have to delete the vm and re-create it, they will not be able to, they will have to pay again. + # so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake + if vm_size["dollars_per_month"] <= account_balance+0.25: + affordable_vm_sizes[key] = vm_size + if request.method == "POST": if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']: return abort(418, f"u want tea") @@ -209,6 +217,8 @@ def create(): errors.append("Size is required") elif size not in vm_sizes: errors.append(f"Invalid size {size}") + elif size not in affordable_vm_sizes: + errors.append(f"Your account must have enough credit to run an {size} for 1 month before you will be allowed to create it") if not os: errors.append("OS is required") @@ -260,13 +270,6 @@ def create(): return redirect(f"{url_for('console.index')}?created={id}") - affordable_vm_sizes = dict() - for key, vm_size in vm_sizes.items(): - # if a user deposits $7.50 and then creates an f1-s vm which costs 7.50 a month, - # then they have to delete the vm and re-create it, they will not be able to, they will have to pay again. - # so for UX it makes a lot of sense to give a small margin of 25 cents for usability sake - if vm_size["dollars_per_month"] <= account_balance+0.25: - affordable_vm_sizes[key] = vm_size for error in errors: flash(error)
    type