Add libvirt-client to Docker image

.. using server-side API tokens

.drone.yml

@@ -0,0 +1,14 @@
+---
+kind: pipeline
+name: publish docker image
+steps:
+  - name: build and publish
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_reg_username_3wc
+      password:
+        from_secret: docker_reg_passwd_3wc
+      repo: 3wordchant/capsul-flask
+      tags: latest
+

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.8-alpine as build
+
+RUN apk add gettext git gcc python3-dev musl-dev \
+    libffi-dev zlib-dev jpeg-dev libjpeg postgresql-dev build-base \
+    --virtual .build-dependencies
+
+RUN mkdir -p /app/{code,venv}
+WORKDIR /app/code
+COPY Pipfile Pipfile.lock /app/code/
+
+RUN python3 -m venv /app/venv
+RUN pip install pipenv setuptools
+ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
+RUN pip install wheel cppy
+# Install dependencies into the virtual environment with Pipenv
+RUN pipenv install --deploy --verbose
+
+FROM python:3.8-alpine
+
+RUN apk add --no-cache libpq libstdc++ libjpeg libvirt-client
+
+COPY . /app/code/
+WORKDIR /app/code
+
+COPY --from=build /app/venv /app/venv
+ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
+
+CMD ["gunicorn", "--bind", "0.0.0.0:5000", "-k", "gevent", "--worker-connections", "1000", "app:app"]
+
+VOLUME /app/code
+
+EXPOSE 5000

capsulflask/__init__.py

@@ -153,7 +153,6 @@ is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line
 app.logger.info(f"is_running_server: {is_running_server}")
 
 if app.config['HUB_MODE_ENABLED']:
-
   if app.config['HUB_MODEL'] == "capsul-flask":
     app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
 
@@ -175,7 +174,9 @@ if app.config['HUB_MODE_ENABLED']:
   from capsulflask import db
   db.init_app(app, is_running_server)
 
-  from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
+  from capsulflask import (
+    auth, landing, console, payment, metrics, cli, hub_api, publicapi, admin
+  )
 
   app.register_blueprint(landing.bp)
   app.register_blueprint(auth.bp)
@@ -185,13 +186,13 @@ if app.config['HUB_MODE_ENABLED']:
   app.register_blueprint(cli.bp)
   app.register_blueprint(hub_api.bp)
   app.register_blueprint(admin.bp)
+  app.register_blueprint(publicapi.bp)
 
   app.add_url_rule("/", endpoint="index")
 
 
 
 if app.config['SPOKE_MODE_ENABLED']:
-
   if app.config['SPOKE_MODEL'] == "shell-scripts":
     app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
   else:
@@ -219,7 +220,6 @@ def override_url_for():
   return dict(url_for=url_for_with_cache_bust)
 
 
-
 def url_for_with_cache_bust(endpoint, **values):
   """
   Add a query parameter based on the hash of the file, this acts as a cache bust
@@ -244,7 +244,3 @@ def url_for_with_cache_bust(endpoint, **values):
       values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
 
   return url_for(endpoint, **values)
-
-
-
-  

capsulflask/auth.py

@@ -1,3 +1,4 @@
+from base64 import b64decode
 import functools
 import re
 
@@ -24,6 +25,15 @@ def account_required(view):
 
     @functools.wraps(view)
     def wrapped_view(**kwargs):
+        api_token = request.headers.get('authorization', None)
+        if api_token is not None:
+            email = get_model().authenticate_token(b64decode(api_token).decode('utf-8'))
+
+            if email is not None:
+                session.clear()
+                session["account"] = email
+                session["csrf-token"] = generate()
+
         if session.get("account") is None or session.get("csrf-token") is None :
             return redirect(url_for("auth.login"))
 

capsulflask/console.py

@@ -1,7 +1,9 @@
+from base64 import b64encode
+from datetime import datetime, timedelta
+import json
 import re
 import sys
-import json
+
-from datetime import datetime, timedelta
 from flask import Blueprint
 from flask import flash
 from flask import current_app
@@ -98,7 +100,6 @@ def index():
 @bp.route("/<string:id>", methods=("GET", "POST"))
 @account_required
 def detail(id):
-
   duration=request.args.get('duration')
   if not duration:
     duration = "5m"
@@ -188,23 +189,13 @@ def detail(id):
       duration=duration
     )
 
-
+def _create(vm_sizes, operating_systems, public_keys_for_account, server_data):
-@bp.route("/create", methods=("GET", "POST"))
-@account_required
-def create():
-  vm_sizes = get_model().vm_sizes_dict()
-  operating_systems = get_model().operating_systems_dict()
-  public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
-  account_balance = get_account_balance(get_vms(), get_payments(), datetime.utcnow())
-  capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
   errors = list()
 
-  if request.method == "POST":
+  size = server_data.get("size")
-    if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
+  os = server_data.get("os")
-      return abort(418, f"u want tea")
+  posted_keys_count = int(server_data.get("ssh_authorized_key_count"))
 
-    size = request.form["size"]
-    os = request.form["os"]
   if not size:
     errors.append("Size is required")
   elif size not in vm_sizes:
@@ -215,15 +206,14 @@ def create():
   elif os not in operating_systems:
     errors.append(f"Invalid os {os}")
 
-    posted_keys_count = int(request.form["ssh_authorized_key_count"])
   posted_keys = list()
 
   if posted_keys_count > 1000:
     errors.append("something went wrong with ssh keys")
   else:
     for i in range(0, posted_keys_count):
-        if f"ssh_key_{i}" in request.form:
+      if f"ssh_key_{i}" in server_data:
-          posted_name = request.form[f"ssh_key_{i}"]
+        posted_name = server_data.get(f"ssh_key_{i}")
         key = None
         for x in public_keys_for_account:
           if x['name'] == posted_name:
@@ -236,7 +226,9 @@ def create():
   if len(posted_keys) == 0:
     errors.append("At least one SSH Public Key is required")
 
-    capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(vm_sizes[size]['memory_mb']*1024*1024)
+  capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(
+    vm_sizes[size]['memory_mb']*1024*1024
+  )
 
   if not capacity_avaliable:
     errors.append("""
@@ -245,19 +237,45 @@ def create():
 
   if len(errors) == 0:
     id = make_capsul_id()
-      # we can't create the vm record in the DB yet because its IP address needs to be allocated first.
+    get_model().create_vm(
-      # so it will be created when the allocation happens inside the hub_api.
+      email=session["account"], 
+      id=id, 
+      size=size, 
+      os=os,
+      ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys))
+    )
     current_app.config["HUB_MODEL"].create(
       email = session["account"],
       id=id,
-        os=os,
-        size=size,
       template_image_file_name=operating_systems[os]['template_image_file_name'],
       vcpus=vm_sizes[size]['vcpus'],
       memory_mb=vm_sizes[size]['memory_mb'],
-        ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys)) 
+      ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys))
     )
+    return id, errors
 
+  return None, errors
+
+@bp.route("/create", methods=("GET", "POST"))
+@account_required
+def create():
+  vm_sizes = get_model().vm_sizes_dict()
+  operating_systems = get_model().operating_systems_dict()
+  public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
+  account_balance = get_account_balance(get_vms(), get_payments(), datetime.utcnow())
+  capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
+
+  if request.method == "POST":
+    if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
+      return abort(418, f"u want tea")
+    id, errors = _create(
+      vm_sizes, 
+      operating_systems,
+      public_keys_for_account,
+      request.form)
+    if len(errors) == 0: 
+      for error in errors:
+        flash(error)
       return redirect(f"{url_for('console.index')}?created={id}")
   
   affordable_vm_sizes = dict()
@@ -268,9 +286,6 @@ def create():
     if vm_size["dollars_per_month"] <= account_balance+0.25:
       affordable_vm_sizes[key] = vm_size
 
-  for error in errors:
-    flash(error)
-
   if not capacity_avaliable:
     current_app.logger.warning(f"when capsul capacity is restored, send an email to {session['account']}")
 
@@ -287,23 +302,25 @@ def create():
     vm_sizes=affordable_vm_sizes
   )
 
-@bp.route("/ssh", methods=("GET", "POST"))
+@bp.route("/keys", methods=("GET", "POST"))
 @account_required
-def ssh_public_keys():
+def ssh_api_keys():
   errors = list()
 
+  token = None
+
   if request.method == "POST":
     if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
       return abort(418, f"u want tea")
       
-    method = request.form["method"]
+    action = request.form["action"]
+
+    if action == 'upload_ssh_key':
       content = None
-    if method == "POST":
       content = request.form["content"].replace("\r", " ").replace("\n", " ").strip()
         
       name = request.form["name"]
       if not name or len(name.strip()) < 1:
-      if method == "POST":
         parts = re.split(" +", content)
         if len(parts) > 2 and len(parts[2].strip()) > 0:
           name = parts[2].strip()
@@ -314,7 +331,6 @@ def ssh_public_keys():
       if not re.match(r"^[0-9A-Za-z_@:. -]+$", name):
         errors.append(f"Key name '{name}' must match \"^[0-9A-Za-z_@:. -]+$\"")
 
-    if method == "POST":
       if not content or len(content.strip()) < 1:
         errors.append("Content is required")
       else:
@@ -327,24 +343,36 @@ def ssh_public_keys():
       if len(errors) == 0:
         get_model().create_ssh_public_key(session["account"], name, content)
 
-    elif method == "DELETE":
+    elif action == "delete_ssh_key":
-
-      if len(errors) == 0:
       get_model().delete_ssh_public_key(session["account"], name)
 
+    elif action == "generate_api_token":
+      name = request.form["name"]
+      if name == '':
+          name = datetime.utcnow().strftime('%y-%m-%d %H:%M:%S')
+      token = b64encode(
+        get_model().generate_api_token(session["account"], name).encode('utf-8')
+      ).decode('utf-8')
+
+    elif action == "delete_api_token":
+      get_model().delete_api_token(session["account"], request.form["id"])
+
   for error in errors:
     flash(error)
 
-  keys_list=list(map(
+  ssh_keys_list=list(map(
     lambda x: dict(name=x['name'], content=f"{x['content'][:20]}...{x['content'][len(x['content'])-20:]}"), 
     get_model().list_ssh_public_keys_for_account(session["account"])
   ))
 
+  api_tokens_list = get_model().list_api_tokens(session["account"])
+
   return render_template(
-    "ssh-public-keys.html", 
+    "keys.html", 
     csrf_token = session["csrf-token"],
-    ssh_public_keys=keys_list, 
+    api_tokens=api_tokens_list, 
-    has_ssh_public_keys=len(keys_list) > 0
+    ssh_public_keys=ssh_keys_list,
+    generated_api_token=token,
   )
 
 def get_vms():
@@ -368,7 +396,6 @@ def get_vm_months_float(vm, as_of):
   return days / average_number_of_days_in_a_month
 
 def get_account_balance(vms, payments, as_of):
-
   vm_cost_dollars = 0.0
   for vm in vms:
     vm_months = get_vm_months_float(vm, as_of)
@@ -381,7 +408,6 @@ def get_account_balance(vms, payments, as_of):
 @bp.route("/account-balance")
 @account_required
 def account_balance():
-
   payment_sessions = get_model().list_payment_sessions_for_account(session['account'])
   for payment_session in payment_sessions:
     if payment_session['type'] == 'btcpay':

capsulflask/db.py

@@ -33,7 +33,7 @@ def init_app(app, is_running_server):
     result = re.search(r"^\d+_(up|down)", filename)
     if not result:
       app.logger.error(f"schemaVersion {filename} must match ^\\d+_(up|down). exiting.")
-      exit(1)
+      continue
     key = result.group()
     with open(join(schemaMigrationsPath, filename), 'rb') as file:
       schemaMigrations[key] = file.read().decode("utf8")
@@ -128,4 +128,3 @@ def close_db(e=None):
   if db_model is not None:
     db_model.cursor.close()
     current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)
-

capsulflask/db_model.py

@@ -1,8 +1,8 @@
-
 import re
 
 # I was never able to get this type hinting to work correctly 
 # from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
+import hashlib
 from nanoid import generate
 from flask import current_app
 from typing import List
@@ -17,7 +17,6 @@ class DBModel:
     self.cursor = cursor
 
 
-
   #     ------    LOGIN     ---------     
 
 
@@ -44,6 +43,16 @@ class DBModel:
 
     return (token, ignoreCaseMatches)
 
+  def authenticate_token(self, token):
+    m = hashlib.md5()
+    m.update(token.encode('utf-8'))
+    hash_token = m.hexdigest()
+    self.cursor.execute("SELECT email FROM api_tokens WHERE token = %s", (hash_token, ))
+    result = self.cursor.fetchall()
+    if len(result) == 1:
+      return result[0]
+    return None
+    
   def consume_token(self, token):
     self.cursor.execute("SELECT email FROM login_tokens WHERE token = %s and created > (NOW() - INTERVAL '20 min')", (token, ))
     row = self.cursor.fetchone()
@@ -132,6 +141,32 @@ class DBModel:
     self.cursor.execute( "DELETE FROM ssh_public_keys where email = %s AND name = %s", (email, name) )
     self.connection.commit()
 
+  def list_api_tokens(self, email):
+    self.cursor.execute(
+      "SELECT id, token, name, created FROM api_tokens WHERE email = %s", 
+      (email, )
+    )
+    return list(map(
+      lambda x: dict(id=x[0], token=x[1], name=x[2], created=x[3]), 
+      self.cursor.fetchall()
+    ))
+
+  def generate_api_token(self, email, name):
+    token = generate()
+    m = hashlib.md5()
+    m.update(token.encode('utf-8'))
+    hash_token = m.hexdigest()
+    self.cursor.execute(
+      "INSERT INTO api_tokens (email, name, token) VALUES (%s, %s, %s)", 
+      (email, name, hash_token)
+    )
+    self.connection.commit()
+    return token
+
+  def delete_api_token(self, email, id_):
+    self.cursor.execute( "DELETE FROM api_tokens where email = %s AND id = %s", (email, id_))
+    self.connection.commit()
+
   def list_vms_for_account(self, email):
     self.cursor.execute(""" 
       SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
@@ -479,8 +514,3 @@ class DBModel:
     #cursor.close()
 
     return to_return
-      
-
-
-
-

capsulflask/publicapi.py

@@ -0,0 +1,40 @@
+import datetime
+
+from flask import Blueprint
+from flask import current_app
+from flask import jsonify
+from flask import request
+from flask import session
+from nanoid import generate
+
+from capsulflask.auth import account_required
+from capsulflask.db import get_model
+
+bp = Blueprint("publicapi", __name__, url_prefix="/api")
+
+@bp.route("/capsul/create", methods=["POST"])
+@account_required
+def capsul_create():
+    email = session["account"]
+    
+    from .console import _create,get_account_balance, get_payments, get_vms
+
+    vm_sizes = get_model().vm_sizes_dict()
+    operating_systems = get_model().operating_systems_dict()
+    public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
+    account_balance = get_account_balance(get_vms(), get_payments(), datetime.datetime.utcnow())
+    capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
+
+    request.json['ssh_authorized_key_count'] = 1
+
+    id, errors = _create(
+      vm_sizes, 
+      operating_systems,
+      public_keys_for_account,
+      request.json)
+
+    if id is not None:
+        return jsonify(
+            id=id,
+        )
+    return jsonify(errors=errors)

capsulflask/schema_migrations/17_down_api_tokens.py

@@ -0,0 +1,2 @@
+DROP TABLE api_keys;
+UPDATE schemaversion SET version = 16;

capsulflask/schema_migrations/17_up_api_tokens.py

@@ -0,0 +1,9 @@
+CREATE TABLE api_tokens (
+  id                 SERIAL PRIMARY KEY,
+  email              TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
+  name               TEXT NOT NULL,
+  created            TIMESTAMP NOT NULL DEFAULT NOW(),
+  token              TEXT NOT NULL
+);
+
+UPDATE schemaversion SET version = 17;

capsulflask/schema_migrations/19_down_api_tokens.py

@@ -0,0 +1,2 @@
+DROP TABLE api_keys;
+UPDATE schemaversion SET version = 16;

capsulflask/schema_migrations/19_up_api_tokens.py

@@ -0,0 +1,9 @@
+CREATE TABLE api_tokens (
+  id                 SERIAL PRIMARY KEY,
+  email              TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
+  name               TEXT NOT NULL,
+  created            TIMESTAMP NOT NULL DEFAULT NOW(),
+  token              TEXT NOT NULL
+);
+
+UPDATE schemaversion SET version = 17;

capsulflask/templates/base.html

@@ -31,7 +31,7 @@
 
     {% if session["account"] %} 
       <a href="/console">Capsuls</a>
-      <a href="/console/ssh">SSH Public Keys</a>
+      <a href="/console/keys">SSH &amp; API Keys</a>
       <a href="/console/account-balance">Account Balance</a>
     {% endif %}
 

capsulflask/templates/capsul-detail.html

@@ -101,7 +101,7 @@
       </div>
       <div class="row justify-start">
         <label class="align" for="ssh_authorized_keys">SSH Authorized Keys</label>
-        <a id="ssh_authorized_keys" href="/console/ssh">{{ vm['ssh_authorized_keys'] }}</a>
+        <a id="ssh_authorized_keys" href="/console/keys">{{ vm['ssh_authorized_keys'] }}</a>
       </div>
       
     </div>

capsulflask/templates/create-capsul.html

@@ -31,7 +31,7 @@
     <p>(At least one month of funding is required)</p>
   {% elif no_ssh_public_keys %}
     <p>You don't have any ssh public keys yet.</p> 
-    <p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
+    <p>You must <a href="/console/keys">upload one</a> before you can create a Capsul.</p>
   {% elif not capacity_avaliable %}
     <p>Host(s) at capacity. No capsuls can be created at this time. sorry. </p> 
   {% else %}

capsulflask/templates/keys.html

@@ -1,17 +1,18 @@
 {% extends 'base.html' %}
 
-{% block title %}SSH Public Keys{% endblock %}
+{% block title %}SSH & API Keys{% endblock %}
 
 {% block content %}
 <div class="row third-margin">
   <h1>SSH PUBLIC KEYS</h1>
 </div>
 <div class="row third-margin"><div>
-  {% if has_ssh_public_keys %} <hr/> {% endif %}
+    {% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
   
   {% for ssh_public_key in ssh_public_keys %}
     <form method="post">
       <input type="hidden" name="method" value="DELETE"></input> 
+      <input type="hidden" name="action" value="delete_ssh_key"></input>
       <input type="hidden" name="name" value="{{ ssh_public_key['name'] }}"></input> 
       <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
       <div class="row">
@@ -22,13 +23,14 @@
     </form>
   {% endfor %}
 
-  {% if has_ssh_public_keys %} <hr/> {% endif %}
+  {% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
 
   <div class="third-margin">
     <h1>UPLOAD A NEW SSH PUBLIC KEY</h1>
   </div>
   <form method="post">
     <input type="hidden" name="method" value="POST"></input>
+    <input type="hidden" name="action" value="upload_ssh_key"></input>
     <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
     <div class="row justify-start">
       <label class="align" for="content">File Contents</label>
@@ -54,6 +56,51 @@
     </div>
   </form>
 </div></div>
+<hr/>
+<div class="row third-margin">
+  <h1>API KEYS</h1>
+</div>
+<div class="row third-margin"><div>
+  {% if generated_api_token %}
+    <hr/>
+    Generated key:
+    <span class="code">{{ generated_api_token }}</span>
+  {% endif %}
+  {% if api_tokens|length >0 %} <hr/>{% endif %}
+  {% for api_token in api_tokens %}
+    <form method="post">
+      <input type="hidden" name="method" value="DELETE"></input> 
+      <input type="hidden" name="action" value="delete_api_token"></input>
+      <input type="hidden" name="id" value="{{ api_token['id'] }}"></input> 
+      <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+      <div class="row">
+        <span class="code">{{ api_token['name'] }}</span>
+        created {{ api_token['created'].strftime("%b %d %Y") }}
+        <input type="submit" value="Delete">
+      </div>
+    </form>
+  {% endfor %}
+  {% if api_tokens|length >0 %} <hr/>{% endif %}
+
+  <div class="third-margin">
+    <h1>GENERATE A NEW API KEY</h1>
+  </div>
+  <form method="post">
+    <input type="hidden" name="method" value="POST"></input>
+    <input type="hidden" name="action" value="generate_api_token"></input>
+    <input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
+    <div class="smalltext">
+      <p>Generate a new API key, to integrate with other systems.</p>
+    </div>
+    <div class="row justify-start">
+      <label class="align" for="name">Key Name</label>
+      <input type="text" id="name" name="name"></input> (defaults to creation time)
+    </div>
+    <div class="row justify-end">
+      <input type="submit" value="Generate">
+    </div>
+  </form>
+</div></div>
 {% endblock %}
 
 {% block pagesource %}/templates/ssh-public-keys.html{% endblock %}

docker-compose.yml

@@ -0,0 +1,29 @@
+---
+version: "3.8"
+
+services:
+  app:
+    image: 3wordchant/capsul-flask:latest
+    build: .
+    volumes:
+      - "./:/app/code"
+    depends_on:
+      - db
+    ports:
+      - "5000:5000"
+    environment:
+      - "POSTGRES_CONNECTION_PARAMETERS=host=db port=5432 user=capsul password=capsul dbname=capsul"
+    # The image uses gunicorn by default, let's override it with Flask's
+    # built-in development server
+    command: ["flask", "run", "-h", "0.0.0.0", "-p", "5000"]
+  db:
+    image: "postgres:9.6.5"
+    volumes:
+      - "postgres:/var/lib/postgresql/data"
+    environment:
+      POSTGRES_USER: capsul
+      POSTGRES_PASSWORD: capsul
+      POSTGRES_DB: capsul
+
+volumes:
+  postgres: