Compare commits

...

18 Commits

Author SHA1 Message Date
3wc 43fb2f9ed3 Add libvirt-client to Docker image
continuous-integration/drone/push Build is passing Details
2021-07-12 23:40:34 +02:00
3wc 9e44f9be0c Merge branch 'webapi' into docker-api
continuous-integration/drone/push Build is passing Details
2021-07-12 22:29:08 +02:00
3wc 64febaceb9 Merge branch 'docker' into docker-api 2021-07-12 22:28:19 +02:00
3wc c4df5c238f Rename migrations 2021-07-12 22:25:06 +02:00
3wc 44874b9056 Updates for upstream IP handling 2021-07-12 22:10:23 +02:00
3wc 630940c186 Add basic "create" API..
.. using server-side API tokens
2021-07-12 22:09:25 +02:00
3wc e9934a8a6b Use Flask server in development 2021-07-12 21:56:14 +02:00
3wc 6db163365c Multi-stage build oh my! 2021-07-12 21:56:14 +02:00
3wc 6b53c80a71 Initial attempt at Docker 2021-07-12 21:56:14 +02:00
3wc 76ec57689f Fix 500 error on Capsuls page
continuous-integration/drone/push Build is passing Details
2021-07-11 23:47:12 +02:00
3wc 49dca12719 Merge branch 'webapi' into docker-api
continuous-integration/drone/push Build is passing Details
2021-07-11 12:36:38 +02:00
3wc 692d936f4a Merge branch 'docker' into docker-api 2021-07-11 12:36:30 +02:00
3wc 8a70e497d1 Use Flask server in development 2021-07-11 12:36:10 +02:00
3wc 6041306a2a Updates for upstream IP handling 2021-07-11 12:35:35 +02:00
3wc 0d9d51c780 Merge branch 'docker' into docker-api
continuous-integration/drone/push Build is passing Details
2021-07-11 00:57:58 +02:00
3wc 647a19bfa7 Multi-stage build oh my! 2021-07-11 00:57:40 +02:00
3wc 541bf7d2d7 Initial attempt at Docker 2021-07-11 00:57:40 +02:00
3wc 3b978b781f Add basic "create" API..
.. using server-side API tokens
2021-07-11 00:57:27 +02:00
17 changed files with 353 additions and 108 deletions

14
.drone.yml Normal file
View File

@ -0,0 +1,14 @@
---
kind: pipeline
name: publish docker image
steps:
- name: build and publish
image: plugins/docker
settings:
username:
from_secret: docker_reg_username_3wc
password:
from_secret: docker_reg_passwd_3wc
repo: 3wordchant/capsul-flask
tags: latest

32
Dockerfile Normal file
View File

@ -0,0 +1,32 @@
FROM python:3.8-alpine as build
RUN apk add gettext git gcc python3-dev musl-dev \
libffi-dev zlib-dev jpeg-dev libjpeg postgresql-dev build-base \
--virtual .build-dependencies
RUN mkdir -p /app/{code,venv}
WORKDIR /app/code
COPY Pipfile Pipfile.lock /app/code/
RUN python3 -m venv /app/venv
RUN pip install pipenv setuptools
ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
RUN pip install wheel cppy
# Install dependencies into the virtual environment with Pipenv
RUN pipenv install --deploy --verbose
FROM python:3.8-alpine
RUN apk add --no-cache libpq libstdc++ libjpeg libvirt-client
COPY . /app/code/
WORKDIR /app/code
COPY --from=build /app/venv /app/venv
ENV PATH="/app/venv/bin:$PATH" VIRTUAL_ENV="/app/venv"
CMD ["gunicorn", "--bind", "0.0.0.0:5000", "-k", "gevent", "--worker-connections", "1000", "app:app"]
VOLUME /app/code
EXPOSE 5000

View File

@ -153,7 +153,6 @@ is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line
app.logger.info(f"is_running_server: {is_running_server}")
if app.config['HUB_MODE_ENABLED']:
if app.config['HUB_MODEL'] == "capsul-flask":
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
@ -175,7 +174,9 @@ if app.config['HUB_MODE_ENABLED']:
from capsulflask import db
db.init_app(app, is_running_server)
from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
from capsulflask import (
auth, landing, console, payment, metrics, cli, hub_api, publicapi, admin
)
app.register_blueprint(landing.bp)
app.register_blueprint(auth.bp)
@ -185,13 +186,13 @@ if app.config['HUB_MODE_ENABLED']:
app.register_blueprint(cli.bp)
app.register_blueprint(hub_api.bp)
app.register_blueprint(admin.bp)
app.register_blueprint(publicapi.bp)
app.add_url_rule("/", endpoint="index")
if app.config['SPOKE_MODE_ENABLED']:
if app.config['SPOKE_MODEL'] == "shell-scripts":
app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
else:
@ -219,7 +220,6 @@ def override_url_for():
return dict(url_for=url_for_with_cache_bust)
def url_for_with_cache_bust(endpoint, **values):
"""
Add a query parameter based on the hash of the file, this acts as a cache bust
@ -244,7 +244,3 @@ def url_for_with_cache_bust(endpoint, **values):
values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
return url_for(endpoint, **values)

View File

@ -1,3 +1,4 @@
from base64 import b64decode
import functools
import re
@ -24,6 +25,15 @@ def account_required(view):
@functools.wraps(view)
def wrapped_view(**kwargs):
api_token = request.headers.get('authorization', None)
if api_token is not None:
email = get_model().authenticate_token(b64decode(api_token).decode('utf-8'))
if email is not None:
session.clear()
session["account"] = email
session["csrf-token"] = generate()
if session.get("account") is None or session.get("csrf-token") is None :
return redirect(url_for("auth.login"))
@ -56,7 +66,7 @@ def login():
if not email:
errors.append("email is required")
elif len(email.strip()) < 6 or email.count('@') != 1 or email.count('.') == 0:
errors.append("enter a valid email address")
errors.append("enter a valid email address")
if len(errors) == 0:
result = get_model().login(email)

View File

@ -1,7 +1,9 @@
from base64 import b64encode
from datetime import datetime, timedelta
import json
import re
import sys
import json
from datetime import datetime, timedelta
from flask import Blueprint
from flask import flash
from flask import current_app
@ -98,7 +100,6 @@ def index():
@bp.route("/<string:id>", methods=("GET", "POST"))
@account_required
def detail(id):
duration=request.args.get('duration')
if not duration:
duration = "5m"
@ -188,6 +189,72 @@ def detail(id):
duration=duration
)
def _create(vm_sizes, operating_systems, public_keys_for_account, server_data):
errors = list()
size = server_data.get("size")
os = server_data.get("os")
posted_keys_count = int(server_data.get("ssh_authorized_key_count"))
if not size:
errors.append("Size is required")
elif size not in vm_sizes:
errors.append(f"Invalid size {size}")
if not os:
errors.append("OS is required")
elif os not in operating_systems:
errors.append(f"Invalid os {os}")
posted_keys = list()
if posted_keys_count > 1000:
errors.append("something went wrong with ssh keys")
else:
for i in range(0, posted_keys_count):
if f"ssh_key_{i}" in server_data:
posted_name = server_data.get(f"ssh_key_{i}")
key = None
for x in public_keys_for_account:
if x['name'] == posted_name:
key = x
if key:
posted_keys.append(key)
else:
errors.append(f"SSH Key \"{posted_name}\" doesn't exist")
if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(
vm_sizes[size]['memory_mb']*1024*1024
)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0:
id = make_capsul_id()
get_model().create_vm(
email=session["account"],
id=id,
size=size,
os=os,
ssh_authorized_keys=list(map(lambda x: x["name"], posted_keys))
)
current_app.config["HUB_MODEL"].create(
email = session["account"],
id=id,
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys))
)
return id, errors
return None, errors
@bp.route("/create", methods=("GET", "POST"))
@account_required
@ -197,67 +264,18 @@ def create():
public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance(get_vms(), get_payments(), datetime.utcnow())
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
errors = list()
if request.method == "POST":
if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
return abort(418, f"u want tea")
size = request.form["size"]
os = request.form["os"]
if not size:
errors.append("Size is required")
elif size not in vm_sizes:
errors.append(f"Invalid size {size}")
if not os:
errors.append("OS is required")
elif os not in operating_systems:
errors.append(f"Invalid os {os}")
posted_keys_count = int(request.form["ssh_authorized_key_count"])
posted_keys = list()
if posted_keys_count > 1000:
errors.append("something went wrong with ssh keys")
else:
for i in range(0, posted_keys_count):
if f"ssh_key_{i}" in request.form:
posted_name = request.form[f"ssh_key_{i}"]
key = None
for x in public_keys_for_account:
if x['name'] == posted_name:
key = x
if key:
posted_keys.append(key)
else:
errors.append(f"SSH Key \"{posted_name}\" doesn't exist")
if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(vm_sizes[size]['memory_mb']*1024*1024)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0:
id = make_capsul_id()
# we can't create the vm record in the DB yet because its IP address needs to be allocated first.
# so it will be created when the allocation happens inside the hub_api.
current_app.config["HUB_MODEL"].create(
email = session["account"],
id=id,
os=os,
size=size,
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys))
)
id, errors = _create(
vm_sizes,
operating_systems,
public_keys_for_account,
request.form)
if len(errors) == 0:
for error in errors:
flash(error)
return redirect(f"{url_for('console.index')}?created={id}")
affordable_vm_sizes = dict()
@ -268,9 +286,6 @@ def create():
if vm_size["dollars_per_month"] <= account_balance+0.25:
affordable_vm_sizes[key] = vm_size
for error in errors:
flash(error)
if not capacity_avaliable:
current_app.logger.warning(f"when capsul capacity is restored, send an email to {session['account']}")
@ -287,23 +302,25 @@ def create():
vm_sizes=affordable_vm_sizes
)
@bp.route("/ssh", methods=("GET", "POST"))
@bp.route("/keys", methods=("GET", "POST"))
@account_required
def ssh_public_keys():
def ssh_api_keys():
errors = list()
token = None
if request.method == "POST":
if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
return abort(418, f"u want tea")
method = request.form["method"]
content = None
if method == "POST":
action = request.form["action"]
if action == 'upload_ssh_key':
content = None
content = request.form["content"].replace("\r", " ").replace("\n", " ").strip()
name = request.form["name"]
if not name or len(name.strip()) < 1:
if method == "POST":
name = request.form["name"]
if not name or len(name.strip()) < 1:
parts = re.split(" +", content)
if len(parts) > 2 and len(parts[2].strip()) > 0:
name = parts[2].strip()
@ -311,10 +328,9 @@ def ssh_public_keys():
name = parts[0].strip()
else:
errors.append("Name is required")
if not re.match(r"^[0-9A-Za-z_@:. -]+$", name):
errors.append(f"Key name '{name}' must match \"^[0-9A-Za-z_@:. -]+$\"")
if not re.match(r"^[0-9A-Za-z_@:. -]+$", name):
errors.append(f"Key name '{name}' must match \"^[0-9A-Za-z_@:. -]+$\"")
if method == "POST":
if not content or len(content.strip()) < 1:
errors.append("Content is required")
else:
@ -327,24 +343,36 @@ def ssh_public_keys():
if len(errors) == 0:
get_model().create_ssh_public_key(session["account"], name, content)
elif method == "DELETE":
elif action == "delete_ssh_key":
get_model().delete_ssh_public_key(session["account"], name)
if len(errors) == 0:
get_model().delete_ssh_public_key(session["account"], name)
elif action == "generate_api_token":
name = request.form["name"]
if name == '':
name = datetime.utcnow().strftime('%y-%m-%d %H:%M:%S')
token = b64encode(
get_model().generate_api_token(session["account"], name).encode('utf-8')
).decode('utf-8')
elif action == "delete_api_token":
get_model().delete_api_token(session["account"], request.form["id"])
for error in errors:
flash(error)
keys_list=list(map(
ssh_keys_list=list(map(
lambda x: dict(name=x['name'], content=f"{x['content'][:20]}...{x['content'][len(x['content'])-20:]}"),
get_model().list_ssh_public_keys_for_account(session["account"])
))
api_tokens_list = get_model().list_api_tokens(session["account"])
return render_template(
"ssh-public-keys.html",
"keys.html",
csrf_token = session["csrf-token"],
ssh_public_keys=keys_list,
has_ssh_public_keys=len(keys_list) > 0
api_tokens=api_tokens_list,
ssh_public_keys=ssh_keys_list,
generated_api_token=token,
)
def get_vms():
@ -368,7 +396,6 @@ def get_vm_months_float(vm, as_of):
return days / average_number_of_days_in_a_month
def get_account_balance(vms, payments, as_of):
vm_cost_dollars = 0.0
for vm in vms:
vm_months = get_vm_months_float(vm, as_of)
@ -381,7 +408,6 @@ def get_account_balance(vms, payments, as_of):
@bp.route("/account-balance")
@account_required
def account_balance():
payment_sessions = get_model().list_payment_sessions_for_account(session['account'])
for payment_session in payment_sessions:
if payment_session['type'] == 'btcpay':

View File

@ -33,7 +33,7 @@ def init_app(app, is_running_server):
result = re.search(r"^\d+_(up|down)", filename)
if not result:
app.logger.error(f"schemaVersion {filename} must match ^\\d+_(up|down). exiting.")
exit(1)
continue
key = result.group()
with open(join(schemaMigrationsPath, filename), 'rb') as file:
schemaMigrations[key] = file.read().decode("utf8")
@ -128,4 +128,3 @@ def close_db(e=None):
if db_model is not None:
db_model.cursor.close()
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)

View File

@ -1,8 +1,8 @@
import re
# I was never able to get this type hinting to work correctly
# from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
import hashlib
from nanoid import generate
from flask import current_app
from typing import List
@ -17,7 +17,6 @@ class DBModel:
self.cursor = cursor
# ------ LOGIN ---------
@ -43,6 +42,16 @@ class DBModel:
self.connection.commit()
return (token, ignoreCaseMatches)
def authenticate_token(self, token):
m = hashlib.md5()
m.update(token.encode('utf-8'))
hash_token = m.hexdigest()
self.cursor.execute("SELECT email FROM api_tokens WHERE token = %s", (hash_token, ))
result = self.cursor.fetchall()
if len(result) == 1:
return result[0]
return None
def consume_token(self, token):
self.cursor.execute("SELECT email FROM login_tokens WHERE token = %s and created > (NOW() - INTERVAL '20 min')", (token, ))
@ -132,6 +141,32 @@ class DBModel:
self.cursor.execute( "DELETE FROM ssh_public_keys where email = %s AND name = %s", (email, name) )
self.connection.commit()
def list_api_tokens(self, email):
self.cursor.execute(
"SELECT id, token, name, created FROM api_tokens WHERE email = %s",
(email, )
)
return list(map(
lambda x: dict(id=x[0], token=x[1], name=x[2], created=x[3]),
self.cursor.fetchall()
))
def generate_api_token(self, email, name):
token = generate()
m = hashlib.md5()
m.update(token.encode('utf-8'))
hash_token = m.hexdigest()
self.cursor.execute(
"INSERT INTO api_tokens (email, name, token) VALUES (%s, %s, %s)",
(email, name, hash_token)
)
self.connection.commit()
return token
def delete_api_token(self, email, id_):
self.cursor.execute( "DELETE FROM api_tokens where email = %s AND id = %s", (email, id_))
self.connection.commit()
def list_vms_for_account(self, email):
self.cursor.execute("""
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
@ -479,8 +514,3 @@ class DBModel:
#cursor.close()
return to_return

40
capsulflask/publicapi.py Normal file
View File

@ -0,0 +1,40 @@
import datetime
from flask import Blueprint
from flask import current_app
from flask import jsonify
from flask import request
from flask import session
from nanoid import generate
from capsulflask.auth import account_required
from capsulflask.db import get_model
bp = Blueprint("publicapi", __name__, url_prefix="/api")
@bp.route("/capsul/create", methods=["POST"])
@account_required
def capsul_create():
email = session["account"]
from .console import _create,get_account_balance, get_payments, get_vms
vm_sizes = get_model().vm_sizes_dict()
operating_systems = get_model().operating_systems_dict()
public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance(get_vms(), get_payments(), datetime.datetime.utcnow())
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
request.json['ssh_authorized_key_count'] = 1
id, errors = _create(
vm_sizes,
operating_systems,
public_keys_for_account,
request.json)
if id is not None:
return jsonify(
id=id,
)
return jsonify(errors=errors)

View File

@ -0,0 +1,2 @@
DROP TABLE api_keys;
UPDATE schemaversion SET version = 16;

View File

@ -0,0 +1,9 @@
CREATE TABLE api_tokens (
id SERIAL PRIMARY KEY,
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
name TEXT NOT NULL,
created TIMESTAMP NOT NULL DEFAULT NOW(),
token TEXT NOT NULL
);
UPDATE schemaversion SET version = 17;

View File

@ -0,0 +1,2 @@
DROP TABLE api_keys;
UPDATE schemaversion SET version = 16;

View File

@ -0,0 +1,9 @@
CREATE TABLE api_tokens (
id SERIAL PRIMARY KEY,
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
name TEXT NOT NULL,
created TIMESTAMP NOT NULL DEFAULT NOW(),
token TEXT NOT NULL
);
UPDATE schemaversion SET version = 17;

View File

@ -31,7 +31,7 @@
{% if session["account"] %}
<a href="/console">Capsuls</a>
<a href="/console/ssh">SSH Public Keys</a>
<a href="/console/keys">SSH &amp; API Keys</a>
<a href="/console/account-balance">Account Balance</a>
{% endif %}

View File

@ -101,7 +101,7 @@
</div>
<div class="row justify-start">
<label class="align" for="ssh_authorized_keys">SSH Authorized Keys</label>
<a id="ssh_authorized_keys" href="/console/ssh">{{ vm['ssh_authorized_keys'] }}</a>
<a id="ssh_authorized_keys" href="/console/keys">{{ vm['ssh_authorized_keys'] }}</a>
</div>
</div>

View File

@ -31,7 +31,7 @@
<p>(At least one month of funding is required)</p>
{% elif no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
<p>You must <a href="/console/keys">upload one</a> before you can create a Capsul.</p>
{% elif not capacity_avaliable %}
<p>Host(s) at capacity. No capsuls can be created at this time. sorry. </p>
{% else %}

View File

@ -1,17 +1,18 @@
{% extends 'base.html' %}
{% block title %}SSH Public Keys{% endblock %}
{% block title %}SSH &amp; API Keys{% endblock %}
{% block content %}
<div class="row third-margin">
<h1>SSH PUBLIC KEYS</h1>
</div>
<div class="row third-margin"><div>
{% if has_ssh_public_keys %} <hr/> {% endif %}
{% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
{% for ssh_public_key in ssh_public_keys %}
<form method="post">
<input type="hidden" name="method" value="DELETE"></input>
<input type="hidden" name="action" value="delete_ssh_key"></input>
<input type="hidden" name="name" value="{{ ssh_public_key['name'] }}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row">
@ -22,13 +23,14 @@
</form>
{% endfor %}
{% if has_ssh_public_keys %} <hr/> {% endif %}
{% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
<div class="third-margin">
<h1>UPLOAD A NEW SSH PUBLIC KEY</h1>
</div>
<form method="post">
<input type="hidden" name="method" value="POST"></input>
<input type="hidden" name="action" value="upload_ssh_key"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row justify-start">
<label class="align" for="content">File Contents</label>
@ -54,6 +56,51 @@
</div>
</form>
</div></div>
<hr/>
<div class="row third-margin">
<h1>API KEYS</h1>
</div>
<div class="row third-margin"><div>
{% if generated_api_token %}
<hr/>
Generated key:
<span class="code">{{ generated_api_token }}</span>
{% endif %}
{% if api_tokens|length >0 %} <hr/>{% endif %}
{% for api_token in api_tokens %}
<form method="post">
<input type="hidden" name="method" value="DELETE"></input>
<input type="hidden" name="action" value="delete_api_token"></input>
<input type="hidden" name="id" value="{{ api_token['id'] }}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row">
<span class="code">{{ api_token['name'] }}</span>
created {{ api_token['created'].strftime("%b %d %Y") }}
<input type="submit" value="Delete">
</div>
</form>
{% endfor %}
{% if api_tokens|length >0 %} <hr/>{% endif %}
<div class="third-margin">
<h1>GENERATE A NEW API KEY</h1>
</div>
<form method="post">
<input type="hidden" name="method" value="POST"></input>
<input type="hidden" name="action" value="generate_api_token"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="smalltext">
<p>Generate a new API key, to integrate with other systems.</p>
</div>
<div class="row justify-start">
<label class="align" for="name">Key Name</label>
<input type="text" id="name" name="name"></input> (defaults to creation time)
</div>
<div class="row justify-end">
<input type="submit" value="Generate">
</div>
</form>
</div></div>
{% endblock %}
{% block pagesource %}/templates/ssh-public-keys.html{% endblock %}

29
docker-compose.yml Normal file
View File

@ -0,0 +1,29 @@
---
version: "3.8"
services:
app:
image: 3wordchant/capsul-flask:latest
build: .
volumes:
- "./:/app/code"
depends_on:
- db
ports:
- "5000:5000"
environment:
- "POSTGRES_CONNECTION_PARAMETERS=host=db port=5432 user=capsul password=capsul dbname=capsul"
# The image uses gunicorn by default, let's override it with Flask's
# built-in development server
command: ["flask", "run", "-h", "0.0.0.0", "-p", "5000"]
db:
image: "postgres:9.6.5"
volumes:
- "postgres:/var/lib/postgresql/data"
environment:
POSTGRES_USER: capsul
POSTGRES_PASSWORD: capsul
POSTGRES_DB: capsul
volumes:
postgres: