Compare commits

...

10 Commits

Author SHA1 Message Date
3wc e1f2318273 Merge branch 'tests' into publicapi-tests 2021-07-17 13:37:43 +02:00
3wc 7b9e9debab Disable VM creation check for the moment 2021-07-17 13:37:28 +02:00
3wc bef379273f Fix botched merge 2021-07-17 13:32:54 +02:00
3wc 82c0c63ff4 Add tests for public API 2021-07-17 13:29:16 +02:00
3wc d5e0e069d1 Merge branch 'tests' into publicapi-tests 2021-07-17 13:28:50 +02:00
3wc b8d149e862 Updates for upstream IP handling 2021-07-17 13:23:43 +02:00
3wc cf42ac5e4d Add basic "create" API..
.. using server-side API tokens
2021-07-17 12:18:16 +02:00
3wc 9823bda4a2 Add SSH key tests 2021-07-17 12:10:28 +02:00
3wc 4c7caa1a38 Initial console tests
NB capsul create isn't working properly, see #83
2021-07-17 11:54:15 +02:00
3wc 665e087bd4 Basic testing using flask-testing
This commit makes it possible to override settings during tests, by
switching capsulflask/__init__.py to a "create_app" pattern, and using
`dotenv_values` instead of `load_dotenv`.

The create_app() method returns a Flask app instance, to give
more control over when to initialise the app. This allows setting
environment variables in test files.

Then, use dotenv_values to override loaded .env variables with ones from
the environment, so that tests can set `POSTGRES_CONNECTION_PARAMETERS`
and `SPOKE_MODEL` (possibly others in future..).

Inital tests for the "landing" pages, and login / activation, are
included.
2021-07-17 10:38:04 +02:00
20 changed files with 725 additions and 270 deletions

View File

@ -9,6 +9,7 @@ blinker = "==1.4"
click = "==7.1.2"
Flask = "==1.1.2"
Flask-Mail = "==0.9.1"
Flask-Testing = "==0.8.1"
gunicorn = "==20.0.4"
isort = "==4.3.21"
itsdangerous = "==1.1.0"

4
app.py
View File

@ -1,2 +1,4 @@
from capsulflask import app
from capsulflask import create_app
create_app()

View File

@ -8,7 +8,7 @@ import requests
import sys
import stripe
from dotenv import load_dotenv, find_dotenv
from dotenv import find_dotenv, dotenv_values
from flask import Flask
from flask_mail import Mail, Message
from flask import render_template
@ -22,106 +22,112 @@ from capsulflask import hub_model, spoke_model, cli
from capsulflask.btcpay import client as btcpay
from capsulflask.http_client import MyHTTPClient
class StdoutMockFlaskMail:
def send(self, message: Message):
current_app.logger.info(f"Email would have been sent if configured:\n\nto: {','.join(message.recipients)}\nsubject: {message.subject}\nbody:\n\n{message.body}\n\n")
load_dotenv(find_dotenv())
def create_app():
app = Flask(__name__)
config = {
**dotenv_values(find_dotenv()),
**os.environ, # override loaded values with environment variables
}
app.config.from_mapping(
BASE_URL=os.environ.get("BASE_URL", default="http://localhost:5000"),
SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"),
HUB_MODE_ENABLED=os.environ.get("HUB_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
SPOKE_MODE_ENABLED=os.environ.get("SPOKE_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
INTERNAL_HTTP_TIMEOUT_SECONDS=os.environ.get("INTERNAL_HTTP_TIMEOUT_SECONDS", default="300"),
HUB_MODEL=os.environ.get("HUB_MODEL", default="capsul-flask"),
SPOKE_MODEL=os.environ.get("SPOKE_MODEL", default="mock"),
LOG_LEVEL=os.environ.get("LOG_LEVEL", default="INFO"),
SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="baikal"),
SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="changeme"),
HUB_TOKEN=os.environ.get("HUB_TOKEN", default="changeme"),
app = Flask(__name__)
# https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
# https://stackoverflow.com/questions/56332906/where-to-put-ssl-certificates-when-trying-to-connect-to-a-remote-database-using
# TLS example: sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=db.example.com port=5432 user=postgres password=dev dbname=postgres
POSTGRES_CONNECTION_PARAMETERS=os.environ.get(
"POSTGRES_CONNECTION_PARAMETERS",
default="host=localhost port=5432 user=postgres password=dev dbname=postgres"
),
app.config.from_mapping(
TESTING=config.get("TESTING", False),
BASE_URL=config.get("BASE_URL", "http://localhost:5000"),
SECRET_KEY=config.get("SECRET_KEY", "dev"),
HUB_MODE_ENABLED=config.get("HUB_MODE_ENABLED", "True").lower() in ['true', '1', 't', 'y', 'yes'],
SPOKE_MODE_ENABLED=config.get("SPOKE_MODE_ENABLED", "True").lower() in ['true', '1', 't', 'y', 'yes'],
INTERNAL_HTTP_TIMEOUT_SECONDS=config.get("INTERNAL_HTTP_TIMEOUT_SECONDS", "300"),
HUB_MODEL=config.get("HUB_MODEL", "capsul-flask"),
SPOKE_MODEL=config.get("SPOKE_MODEL", "mock"),
LOG_LEVEL=config.get("LOG_LEVEL", "INFO"),
SPOKE_HOST_ID=config.get("SPOKE_HOST_ID", "baikal"),
SPOKE_HOST_TOKEN=config.get("SPOKE_HOST_TOKEN", "changeme"),
HUB_TOKEN=config.get("HUB_TOKEN", "changeme"),
DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
# https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
# https://stackoverflow.com/questions/56332906/where-to-put-ssl-certificates-when-trying-to-connect-to-a-remote-database-using
# TLS example: sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=db.example.com port=5432 user=postgres password=dev dbname=postgres
POSTGRES_CONNECTION_PARAMETERS=config.get(
"POSTGRES_CONNECTION_PARAMETERS",
"host=localhost port=5432 user=postgres password=dev dbname=postgres"
),
MAIL_SERVER=os.environ.get("MAIL_SERVER", default=""),
MAIL_PORT=os.environ.get("MAIL_PORT", default="465"),
MAIL_USE_TLS=os.environ.get("MAIL_USE_TLS", default="False").lower() in ['true', '1', 't', 'y', 'yes'],
MAIL_USE_SSL=os.environ.get("MAIL_USE_SSL", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
MAIL_USERNAME=os.environ.get("MAIL_USERNAME", default=""),
MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
ADMIN_EMAIL_ADDRESSES=os.environ.get("ADMIN_EMAIL_ADDRESSES", default="ops@cyberia.club"),
ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="forest.n.johnson@gmail.com,capsul@cyberia.club"),
DATABASE_SCHEMA=config.get("DATABASE_SCHEMA", "public"),
PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
MAIL_SERVER=config.get("MAIL_SERVER", ""),
MAIL_PORT=config.get("MAIL_PORT", "465"),
MAIL_USE_TLS=config.get("MAIL_USE_TLS", "False").lower() in ['true', '1', 't', 'y', 'yes'],
MAIL_USE_SSL=config.get("MAIL_USE_SSL", "True").lower() in ['true', '1', 't', 'y', 'yes'],
MAIL_USERNAME=config.get("MAIL_USERNAME", ""),
MAIL_PASSWORD=config.get("MAIL_PASSWORD", ""),
MAIL_DEFAULT_SENDER=config.get("MAIL_DEFAULT_SENDER", "no-reply@capsul.org"),
ADMIN_EMAIL_ADDRESSES=config.get("ADMIN_EMAIL_ADDRESSES", "ops@cyberia.club"),
ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=config.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", "forest.n.johnson@gmail.com,capsul@cyberia.club"),
STRIPE_API_VERSION=os.environ.get("STRIPE_API_VERSION", default="2020-03-02"),
STRIPE_SECRET_KEY=os.environ.get("STRIPE_SECRET_KEY", default=""),
STRIPE_PUBLISHABLE_KEY=os.environ.get("STRIPE_PUBLISHABLE_KEY", default=""),
#STRIPE_WEBHOOK_SECRET=os.environ.get("STRIPE_WEBHOOK_SECRET", default="")
PROMETHEUS_URL=config.get("PROMETHEUS_URL", "https://prometheus.cyberia.club"),
BTCPAY_PRIVATE_KEY=os.environ.get("BTCPAY_PRIVATE_KEY", default="").replace("\\n", "\n"),
BTCPAY_URL=os.environ.get("BTCPAY_URL", default="https://btcpay.cyberia.club")
)
STRIPE_API_VERSION=config.get("STRIPE_API_VERSION", "2020-03-02"),
STRIPE_SECRET_KEY=config.get("STRIPE_SECRET_KEY", ""),
STRIPE_PUBLISHABLE_KEY=config.get("STRIPE_PUBLISHABLE_KEY", ""),
#STRIPE_WEBHOOK_SECRET=config.get("STRIPE_WEBHOOK_SECRET", "")
app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL'])
BTCPAY_PRIVATE_KEY=config.get("BTCPAY_PRIVATE_KEY", "").replace("\\n", "\n"),
BTCPAY_URL=config.get("BTCPAY_URL", "https://btcpay.cyberia.club")
)
class SetLogLevelToDebugForHeartbeatRelatedMessagesFilter(logging.Filter):
def isHeartbeatRelatedString(self, thing):
# thing_string = "<error>"
is_in_string = False
try:
thing_string = "%s" % thing
is_in_string = 'heartbeat-task' in thing_string or 'hub/heartbeat' in thing_string or 'spoke/heartbeat' in thing_string
except:
pass
# self.warning("isHeartbeatRelatedString(%s): %s", thing_string, is_in_string )
return is_in_string
app.config['HUB_URL'] = config.get("HUB_URL", app.config['BASE_URL'])
class SetLogLevelToDebugForHeartbeatRelatedMessagesFilter(logging.Filter):
def isHeartbeatRelatedString(self, thing):
# thing_string = "<error>"
is_in_string = False
try:
thing_string = "%s" % thing
is_in_string = 'heartbeat-task' in thing_string or 'hub/heartbeat' in thing_string or 'spoke/heartbeat' in thing_string
except:
pass
# self.warning("isHeartbeatRelatedString(%s): %s", thing_string, is_in_string )
return is_in_string
def filter(self, record):
if app.config['LOG_LEVEL'] == "DEBUG":
return True
if self.isHeartbeatRelatedString(record.msg):
return False
for arg in record.args:
if self.isHeartbeatRelatedString(arg):
return False
def filter(self, record):
if app.config['LOG_LEVEL'] == "DEBUG":
return True
if self.isHeartbeatRelatedString(record.msg):
return False
for arg in record.args:
if self.isHeartbeatRelatedString(arg):
return False
return True
logging_dict_config({
'version': 1,
'formatters': {'default': {
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}},
'filters': {
'setLogLevelToDebugForHeartbeatRelatedMessages': {
'()': SetLogLevelToDebugForHeartbeatRelatedMessagesFilter,
logging_dict_config({
'version': 1,
'formatters': {'default': {
'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
}},
'filters': {
'setLogLevelToDebugForHeartbeatRelatedMessages': {
'()': SetLogLevelToDebugForHeartbeatRelatedMessagesFilter,
}
},
'handlers': {'wsgi': {
'class': 'logging.StreamHandler',
'stream': 'ext://flask.logging.wsgi_errors_stream',
'formatter': 'default',
'filters': ['setLogLevelToDebugForHeartbeatRelatedMessages']
}},
'root': {
'level': app.config['LOG_LEVEL'],
'handlers': ['wsgi']
}
},
'handlers': {'wsgi': {
'class': 'logging.StreamHandler',
'stream': 'ext://flask.logging.wsgi_errors_stream',
'formatter': 'default',
'filters': ['setLogLevelToDebugForHeartbeatRelatedMessages']
}},
'root': {
'level': app.config['LOG_LEVEL'],
'handlers': ['wsgi']
}
})
})
# app.logger.critical("critical")
# app.logger.error("error")
@ -129,122 +135,122 @@ logging_dict_config({
# app.logger.info("info")
# app.logger.debug("debug")
stripe.api_key = app.config['STRIPE_SECRET_KEY']
stripe.api_version = app.config['STRIPE_API_VERSION']
stripe.api_key = app.config['STRIPE_SECRET_KEY']
stripe.api_version = app.config['STRIPE_API_VERSION']
if app.config['MAIL_SERVER'] != "":
app.config['FLASK_MAIL_INSTANCE'] = Mail(app)
else:
app.logger.warning("No MAIL_SERVER configured. capsul will simply print emails to stdout.")
app.config['FLASK_MAIL_INSTANCE'] = StdoutMockFlaskMail()
if app.config['MAIL_SERVER'] != "":
app.config['FLASK_MAIL_INSTANCE'] = Mail(app)
else:
app.logger.warning("No MAIL_SERVER configured. capsul will simply print emails to stdout.")
app.config['FLASK_MAIL_INSTANCE'] = StdoutMockFlaskMail()
app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
try:
app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
except:
app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
try:
app.config['BTCPAY_CLIENT'] = btcpay.Client(api_uri=app.config['BTCPAY_URL'], pem=app.config['BTCPAY_PRIVATE_KEY'])
except:
app.logger.warning("unable to create btcpay client. Capsul will work fine except cryptocurrency payments will not work. The error was: " + my_exec_info_message(sys.exc_info()))
# only start the scheduler and attempt to migrate the database if we are running the app.
# otherwise we are running a CLI command.
command_line = ' '.join(sys.argv)
is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
command_line = ' '.join(sys.argv)
is_running_server = (
('flask run' in command_line) or
('gunicorn' in command_line) or
('test' in command_line)
)
app.logger.info(f"is_running_server: {is_running_server}")
app.logger.info(f"is_running_server: {is_running_server}")
if app.config['HUB_MODE_ENABLED']:
if app.config['HUB_MODE_ENABLED']:
if app.config['HUB_MODEL'] == "capsul-flask":
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
if app.config['HUB_MODEL'] == "capsul-flask":
app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub()
# debug mode (flask reloader) runs two copies of the app. When running in debug mode,
# we only want to start the scheduler one time.
if is_running_server and (not app.debug or config.get('WERKZEUG_RUN_MAIN') == 'true'):
scheduler = BackgroundScheduler()
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
scheduler.add_job(name="heartbeat-task", func=heartbeat_task, trigger="interval", seconds=5)
scheduler.start()
# debug mode (flask reloader) runs two copies of the app. When running in debug mode,
# we only want to start the scheduler one time.
if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
scheduler = BackgroundScheduler()
heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
heartbeat_task_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=heartbeat_task_headers)
scheduler.add_job(name="heartbeat-task", func=heartbeat_task, trigger="interval", seconds=5)
scheduler.start()
atexit.register(lambda: scheduler.shutdown())
atexit.register(lambda: scheduler.shutdown())
else:
app.config['HUB_MODEL'] = hub_model.MockHub()
else:
app.config['HUB_MODEL'] = hub_model.MockHub()
from capsulflask import db
db.init_app(app, is_running_server)
from capsulflask import db
db.init_app(app, is_running_server)
from capsulflask import (
auth, landing, console, payment, metrics, cli, hub_api, publicapi, admin
)
from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
app.register_blueprint(landing.bp)
app.register_blueprint(auth.bp)
app.register_blueprint(landing.bp)
app.register_blueprint(console.bp)
app.register_blueprint(payment.bp)
app.register_blueprint(metrics.bp)
app.register_blueprint(cli.bp)
app.register_blueprint(hub_api.bp)
app.register_blueprint(admin.bp)
app.register_blueprint(publicapi.bp)
app.add_url_rule("/", endpoint="index")
if app.config['SPOKE_MODE_ENABLED']:
if app.config['SPOKE_MODEL'] == "shell-scripts":
app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
else:
app.config['SPOKE_MODEL'] = spoke_model.MockSpoke()
from capsulflask import spoke_api
app.register_blueprint(spoke_api.bp)
@app.after_request
def security_headers(response):
response.headers['X-Frame-Options'] = 'SAMEORIGIN'
if 'Content-Security-Policy' not in response.headers:
response.headers['Content-Security-Policy'] = "default-src 'self'"
response.headers['X-Content-Type-Options'] = 'nosniff'
return response
if app.config['SPOKE_MODE_ENABLED']:
if app.config['SPOKE_MODEL'] == "shell-scripts":
app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke()
else:
app.config['SPOKE_MODEL'] = spoke_model.MockSpoke()
from capsulflask import spoke_api
app.register_blueprint(spoke_api.bp)
@app.after_request
def security_headers(response):
response.headers['X-Frame-Options'] = 'SAMEORIGIN'
if 'Content-Security-Policy' not in response.headers:
response.headers['Content-Security-Policy'] = "default-src 'self'"
response.headers['X-Content-Type-Options'] = 'nosniff'
return response
@app.context_processor
def override_url_for():
"""
override the url_for function built into flask
with our own custom implementation that busts the cache correctly when files change
"""
return dict(url_for=url_for_with_cache_bust)
@app.context_processor
def override_url_for():
"""
override the url_for function built into flask
with our own custom implementation that busts the cache correctly when files change
"""
return dict(url_for=url_for_with_cache_bust)
def url_for_with_cache_bust(endpoint, **values):
"""
Add a query parameter based on the hash of the file, this acts as a cache bust
"""
if endpoint == 'static':
filename = values.get('filename', None)
if filename:
if 'STATIC_FILE_HASH_CACHE' not in current_app.config:
current_app.config['STATIC_FILE_HASH_CACHE'] = dict()
if filename not in current_app.config['STATIC_FILE_HASH_CACHE']:
filepath = os.path.join(current_app.root_path, endpoint, filename)
#print(filepath)
if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
with open(filepath, 'rb') as file:
hasher = hashlib.md5()
hasher.update(file.read())
current_app.config['STATIC_FILE_HASH_CACHE'][filename] = hasher.hexdigest()[-6:]
def url_for_with_cache_bust(endpoint, **values):
"""
Add a query parameter based on the hash of the file, this acts as a cache bust
"""
if endpoint == 'static':
filename = values.get('filename', None)
if filename:
if 'STATIC_FILE_HASH_CACHE' not in current_app.config:
current_app.config['STATIC_FILE_HASH_CACHE'] = dict()
if filename not in current_app.config['STATIC_FILE_HASH_CACHE']:
filepath = os.path.join(current_app.root_path, endpoint, filename)
#print(filepath)
if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
with open(filepath, 'rb') as file:
hasher = hashlib.md5()
hasher.update(file.read())
current_app.config['STATIC_FILE_HASH_CACHE'][filename] = hasher.hexdigest()[-6:]
values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
return url_for(endpoint, **values)
return url_for(endpoint, **values)
return app

View File

@ -1,3 +1,4 @@
from base64 import b64decode
import functools
import re
@ -24,6 +25,15 @@ def account_required(view):
@functools.wraps(view)
def wrapped_view(**kwargs):
api_token = request.headers.get('authorization', None)
if api_token is not None:
email = get_model().authenticate_token(b64decode(api_token).decode('utf-8'))
if email is not None:
session.clear()
session["account"] = email
session["csrf-token"] = generate()
if session.get("account") is None or session.get("csrf-token") is None :
return redirect(url_for("auth.login"))
@ -31,7 +41,6 @@ def account_required(view):
return wrapped_view
def admin_account_required(view):
"""View decorator that redirects non-admin users to the login page."""
@ -56,7 +65,7 @@ def login():
if not email:
errors.append("email is required")
elif len(email.strip()) < 6 or email.count('@') != 1 or email.count('.') == 0:
errors.append("enter a valid email address")
errors.append("enter a valid email address")
if len(errors) == 0:
result = get_model().login(email)

View File

@ -1,7 +1,9 @@
from base64 import b64encode
from datetime import datetime, timedelta
import json
import re
import sys
import json
from datetime import datetime, timedelta
from flask import Blueprint
from flask import flash
from flask import current_app
@ -98,7 +100,6 @@ def index():
@bp.route("/<string:id>", methods=("GET", "POST"))
@account_required
def detail(id):
duration=request.args.get('duration')
if not duration:
duration = "5m"
@ -188,6 +189,67 @@ def detail(id):
duration=duration
)
def _create(vm_sizes, operating_systems, public_keys_for_account, server_data):
errors = list()
size = server_data.get("size")
os = server_data.get("os")
posted_keys_count = int(server_data.get("ssh_authorized_key_count"))
if not size:
errors.append("Size is required")
elif size not in vm_sizes:
errors.append(f"Invalid size {size}")
if not os:
errors.append("OS is required")
elif os not in operating_systems:
errors.append(f"Invalid os {os}")
posted_keys = list()
if posted_keys_count > 1000:
errors.append("something went wrong with ssh keys")
else:
for i in range(0, posted_keys_count):
if f"ssh_key_{i}" in server_data:
posted_name = server_data.get(f"ssh_key_{i}")
key = None
for x in public_keys_for_account:
if x['name'] == posted_name:
key = x
if key:
posted_keys.append(key)
else:
errors.append(f"SSH Key \"{posted_name}\" doesn't exist")
if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(
vm_sizes[size]['memory_mb']*1024*1024
)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0:
id = make_capsul_id()
current_app.config["HUB_MODEL"].create(
email = session["account"],
id=id,
os=os,
size=size,
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: x["content"], posted_keys))
)
return id, errors
return None, errors
@bp.route("/create", methods=("GET", "POST"))
@account_required
@ -202,62 +264,12 @@ def create():
if request.method == "POST":
if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
return abort(418, f"u want tea")
size = request.form["size"]
os = request.form["os"]
if not size:
errors.append("Size is required")
elif size not in vm_sizes:
errors.append(f"Invalid size {size}")
if not os:
errors.append("OS is required")
elif os not in operating_systems:
errors.append(f"Invalid os {os}")
posted_keys_count = int(request.form["ssh_authorized_key_count"])
posted_keys = list()
if posted_keys_count > 1000:
errors.append("something went wrong with ssh keys")
else:
for i in range(0, posted_keys_count):
if f"ssh_key_{i}" in request.form:
posted_name = request.form[f"ssh_key_{i}"]
key = None
for x in public_keys_for_account:
if x['name'] == posted_name:
key = x
if key:
posted_keys.append(key)
else:
errors.append(f"SSH Key \"{posted_name}\" doesn't exist")
if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(vm_sizes[size]['memory_mb']*1024*1024)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0:
id = make_capsul_id()
# we can't create the vm record in the DB yet because its IP address needs to be allocated first.
# so it will be created when the allocation happens inside the hub_api.
current_app.config["HUB_MODEL"].create(
email = session["account"],
id=id,
os=os,
size=size,
template_image_file_name=operating_systems[os]['template_image_file_name'],
vcpus=vm_sizes[size]['vcpus'],
memory_mb=vm_sizes[size]['memory_mb'],
ssh_authorized_keys=list(map(lambda x: dict(name=x['name'], content=x['content']), posted_keys))
)
id, errors = _create(
vm_sizes,
operating_systems,
public_keys_for_account,
request.form)
if len(errors) == 0:
return redirect(f"{url_for('console.index')}?created={id}")
affordable_vm_sizes = dict()
@ -287,23 +299,25 @@ def create():
vm_sizes=affordable_vm_sizes
)
@bp.route("/ssh", methods=("GET", "POST"))
@bp.route("/keys", methods=("GET", "POST"))
@account_required
def ssh_public_keys():
def ssh_api_keys():
errors = list()
token = None
if request.method == "POST":
if "csrf-token" not in request.form or request.form['csrf-token'] != session['csrf-token']:
return abort(418, f"u want tea")
method = request.form["method"]
content = None
if method == "POST":
action = request.form["action"]
if action == 'upload_ssh_key':
content = None
content = request.form["content"].replace("\r", " ").replace("\n", " ").strip()
name = request.form["name"]
if not name or len(name.strip()) < 1:
if method == "POST":
name = request.form["name"]
if not name or len(name.strip()) < 1:
parts = re.split(" +", content)
if len(parts) > 2 and len(parts[2].strip()) > 0:
name = parts[2].strip()
@ -311,10 +325,9 @@ def ssh_public_keys():
name = parts[0].strip()
else:
errors.append("Name is required")
if not re.match(r"^[0-9A-Za-z_@:. -]+$", name):
errors.append(f"Key name '{name}' must match \"^[0-9A-Za-z_@:. -]+$\"")
if not re.match(r"^[0-9A-Za-z_@:. -]+$", name):
errors.append(f"Key name '{name}' must match \"^[0-9A-Za-z_@:. -]+$\"")
if method == "POST":
if not content or len(content.strip()) < 1:
errors.append("Content is required")
else:
@ -327,24 +340,36 @@ def ssh_public_keys():
if len(errors) == 0:
get_model().create_ssh_public_key(session["account"], name, content)
elif method == "DELETE":
elif action == "delete_ssh_key":
get_model().delete_ssh_public_key(session["account"], name)
if len(errors) == 0:
get_model().delete_ssh_public_key(session["account"], name)
elif action == "generate_api_token":
name = request.form["name"]
if name == '':
name = datetime.utcnow().strftime('%y-%m-%d %H:%M:%S')
token = b64encode(
get_model().generate_api_token(session["account"], name).encode('utf-8')
).decode('utf-8')
elif action == "delete_api_token":
get_model().delete_api_token(session["account"], request.form["id"])
for error in errors:
flash(error)
keys_list=list(map(
ssh_keys_list=list(map(
lambda x: dict(name=x['name'], content=f"{x['content'][:20]}...{x['content'][len(x['content'])-20:]}"),
get_model().list_ssh_public_keys_for_account(session["account"])
))
api_tokens_list = get_model().list_api_tokens(session["account"])
return render_template(
"ssh-public-keys.html",
"keys.html",
csrf_token = session["csrf-token"],
ssh_public_keys=keys_list,
has_ssh_public_keys=len(keys_list) > 0
api_tokens=api_tokens_list,
ssh_public_keys=ssh_keys_list,
generated_api_token=token,
)
def get_vms():
@ -368,7 +393,6 @@ def get_vm_months_float(vm, as_of):
return days / average_number_of_days_in_a_month
def get_account_balance(vms, payments, as_of):
vm_cost_dollars = 0.0
for vm in vms:
vm_months = get_vm_months_float(vm, as_of)
@ -381,7 +405,6 @@ def get_account_balance(vms, payments, as_of):
@bp.route("/account-balance")
@account_required
def account_balance():
payment_sessions = get_model().list_payment_sessions_for_account(session['account'])
for payment_session in payment_sessions:
if payment_session['type'] == 'btcpay':

View File

@ -33,7 +33,7 @@ def init_app(app, is_running_server):
result = re.search(r"^\d+_(up|down)", filename)
if not result:
app.logger.error(f"schemaVersion {filename} must match ^\\d+_(up|down). exiting.")
exit(1)
continue
key = result.group()
with open(join(schemaMigrationsPath, filename), 'rb') as file:
schemaMigrations[key] = file.read().decode("utf8")
@ -43,7 +43,7 @@ def init_app(app, is_running_server):
hasSchemaVersionTable = False
actionWasTaken = False
schemaVersion = 0
desiredSchemaVersion = 18
desiredSchemaVersion = 19
cursor = connection.cursor()
@ -128,4 +128,3 @@ def close_db(e=None):
if db_model is not None:
db_model.cursor.close()
current_app.config['PSYCOPG2_CONNECTION_POOL'].putconn(db_model.connection)

View File

@ -1,8 +1,8 @@
import re
# I was never able to get this type hinting to work correctly
# from psycopg2.extensions import connection as Psycopg2Connection, cursor as Psycopg2Cursor
import hashlib
from nanoid import generate
from flask import current_app
from typing import List
@ -17,7 +17,6 @@ class DBModel:
self.cursor = cursor
# ------ LOGIN ---------
@ -43,6 +42,16 @@ class DBModel:
self.connection.commit()
return (token, ignoreCaseMatches)
def authenticate_token(self, token):
m = hashlib.md5()
m.update(token.encode('utf-8'))
hash_token = m.hexdigest()
self.cursor.execute("SELECT email FROM api_tokens WHERE token = %s", (hash_token, ))
result = self.cursor.fetchall()
if len(result) == 1:
return result[0]
return None
def consume_token(self, token):
self.cursor.execute("SELECT email FROM login_tokens WHERE token = %s and created > (NOW() - INTERVAL '20 min')", (token, ))
@ -132,6 +141,32 @@ class DBModel:
self.cursor.execute( "DELETE FROM ssh_public_keys where email = %s AND name = %s", (email, name) )
self.connection.commit()
def list_api_tokens(self, email):
self.cursor.execute(
"SELECT id, token, name, created FROM api_tokens WHERE email = %s",
(email, )
)
return list(map(
lambda x: dict(id=x[0], token=x[1], name=x[2], created=x[3]),
self.cursor.fetchall()
))
def generate_api_token(self, email, name):
token = generate()
m = hashlib.md5()
m.update(token.encode('utf-8'))
hash_token = m.hexdigest()
self.cursor.execute(
"INSERT INTO api_tokens (email, name, token) VALUES (%s, %s, %s)",
(email, name, hash_token)
)
self.connection.commit()
return token
def delete_api_token(self, email, id_):
self.cursor.execute( "DELETE FROM api_tokens where email = %s AND id = %s", (email, id_))
self.connection.commit()
def list_vms_for_account(self, email):
self.cursor.execute("""
SELECT vms.id, vms.public_ipv4, vms.public_ipv6, vms.size, vms.os, vms.created, vms.deleted, vm_sizes.dollars_per_month
@ -479,8 +514,3 @@ class DBModel:
#cursor.close()
return to_return

40
capsulflask/publicapi.py Normal file
View File

@ -0,0 +1,40 @@
import datetime
from flask import Blueprint
from flask import current_app
from flask import jsonify
from flask import request
from flask import session
from nanoid import generate
from capsulflask.auth import account_required
from capsulflask.db import get_model
bp = Blueprint("publicapi", __name__, url_prefix="/api")
@bp.route("/capsul/create", methods=["POST"])
@account_required
def capsul_create():
email = session["account"]
from .console import _create,get_account_balance, get_payments, get_vms
vm_sizes = get_model().vm_sizes_dict()
operating_systems = get_model().operating_systems_dict()
public_keys_for_account = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance(get_vms(), get_payments(), datetime.datetime.utcnow())
capacity_avaliable = current_app.config["HUB_MODEL"].capacity_avaliable(512*1024*1024)
request.json['ssh_authorized_key_count'] = 1
id, errors = _create(
vm_sizes,
operating_systems,
public_keys_for_account,
request.json)
if id is not None:
return jsonify(
id=id,
)
return jsonify(errors=errors)

View File

@ -0,0 +1,2 @@
DROP TABLE api_keys;
UPDATE schemaversion SET version = 18;

View File

@ -0,0 +1,9 @@
CREATE TABLE api_tokens (
id SERIAL PRIMARY KEY,
email TEXT REFERENCES accounts(email) ON DELETE RESTRICT,
name TEXT NOT NULL,
created TIMESTAMP NOT NULL DEFAULT NOW(),
token TEXT NOT NULL
);
UPDATE schemaversion SET version = 19;

View File

@ -31,7 +31,7 @@
{% if session["account"] %}
<a href="/console">Capsuls</a>
<a href="/console/ssh">SSH Public Keys</a>
<a href="/console/keys">SSH &amp; API Keys</a>
<a href="/console/account-balance">Account Balance</a>
{% endif %}

View File

@ -101,7 +101,7 @@
</div>
<div class="row justify-start">
<label class="align" for="ssh_authorized_keys">SSH Authorized Keys</label>
<a id="ssh_authorized_keys" href="/console/ssh">{{ vm['ssh_authorized_keys'] }}</a>
<a id="ssh_authorized_keys" href="/console/keys">{{ vm['ssh_authorized_keys'] }}</a>
</div>
</div>

View File

@ -31,7 +31,7 @@
<p>(At least one month of funding is required)</p>
{% elif no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
<p>You must <a href="/console/keys">upload one</a> before you can create a Capsul.</p>
{% elif not capacity_avaliable %}
<p>Host(s) at capacity. No capsuls can be created at this time. sorry. </p>
{% else %}

View File

@ -1,17 +1,18 @@
{% extends 'base.html' %}
{% block title %}SSH Public Keys{% endblock %}
{% block title %}SSH &amp; API Keys{% endblock %}
{% block content %}
<div class="row third-margin">
<h1>SSH PUBLIC KEYS</h1>
</div>
<div class="row third-margin"><div>
{% if has_ssh_public_keys %} <hr/> {% endif %}
{% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
{% for ssh_public_key in ssh_public_keys %}
<form method="post">
<input type="hidden" name="method" value="DELETE"></input>
<input type="hidden" name="action" value="delete_ssh_key"></input>
<input type="hidden" name="name" value="{{ ssh_public_key['name'] }}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row">
@ -22,13 +23,14 @@
</form>
{% endfor %}
{% if has_ssh_public_keys %} <hr/> {% endif %}
{% if ssh_public_keys|length > 0 %} <hr/> {% endif %}
<div class="third-margin">
<h1>UPLOAD A NEW SSH PUBLIC KEY</h1>
</div>
<form method="post">
<input type="hidden" name="method" value="POST"></input>
<input type="hidden" name="action" value="upload_ssh_key"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row justify-start">
<label class="align" for="content">File Contents</label>
@ -54,6 +56,51 @@
</div>
</form>
</div></div>
<hr/>
<div class="row third-margin">
<h1>API KEYS</h1>
</div>
<div class="row third-margin"><div>
{% if generated_api_token %}
<hr/>
Generated key:
<span class="code">{{ generated_api_token }}</span>
{% endif %}
{% if api_tokens|length >0 %} <hr/>{% endif %}
{% for api_token in api_tokens %}
<form method="post">
<input type="hidden" name="method" value="DELETE"></input>
<input type="hidden" name="action" value="delete_api_token"></input>
<input type="hidden" name="id" value="{{ api_token['id'] }}"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="row">
<span class="code">{{ api_token['name'] }}</span>
created {{ api_token['created'].strftime("%b %d %Y") }}
<input type="submit" value="Delete">
</div>
</form>
{% endfor %}
{% if api_tokens|length >0 %} <hr/>{% endif %}
<div class="third-margin">
<h1>GENERATE A NEW API KEY</h1>
</div>
<form method="post">
<input type="hidden" name="method" value="POST"></input>
<input type="hidden" name="action" value="generate_api_token"></input>
<input type="hidden" name="csrf-token" value="{{ csrf_token }}"/>
<div class="smalltext">
<p>Generate a new API key, to integrate with other systems.</p>
</div>
<div class="row justify-start">
<label class="align" for="name">Key Name</label>
<input type="text" id="name" name="name"></input> (defaults to creation time)
</div>
<div class="row justify-end">
<input type="submit" value="Generate">
</div>
</form>
</div></div>
{% endblock %}
{% block pagesource %}/templates/ssh-public-keys.html{% endblock %}

View File

View File

@ -0,0 +1,23 @@
from flask import url_for, session
from capsulflask.db import get_model
from capsulflask.tests_base import BaseTestCase
class LoginTests(BaseTestCase):
render_templates = False
def test_login_request(self):
with self.client as client:
response = client.get(url_for("auth.login"))
self.assert_200(response)
# FIXME test generated login link
def test_login_magiclink(self):
token, ignoreCaseMatches = get_model().login('test@example.com')
with self.client as client:
response = client.get(url_for("auth.magiclink", token=token))
self.assertRedirects(response, url_for("console.index"))
self.assertEqual(session['account'], 'test@example.com')

View File

@ -0,0 +1,170 @@
from unittest.mock import patch
from flask import url_for
from capsulflask.hub_model import MockHub
from capsulflask.db import get_model
from capsulflask.tests_base import BaseTestCase
class ConsoleTests(BaseTestCase):
capsul_data = {
"size": "f1-xs",
"os": "debian10",
"ssh_authorized_key_count": 1,
"ssh_key_0": "key"
}
ssh_key_data = {
"name": "key2",
"action": "upload_ssh_key",
"content": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDntq1t8Ddsa2q4p+PM7W4CLYYmxakokRRVLlf7AQlsTJFPsgBe9u0zuoOaKDMkBr0dlnuLm4Eub1Mj+BrdqAokto0YDiAnxUKRuYQKuHySKK8bLkisi2k47jGBDikx/jihgiuFTawo1mYsJJepC7PPwZGsoCImJEgq1L+ug0p3Zrj3QkUx4h25MpCSs2yvfgWjDyN8hEC76O42P+4ETezYrzrd1Kj26hdzHRnrxygvIUOtfau+5ydlaz8xQBEPrEY6/+pKDuwtXg1pBL7GmoUxBXVfHQSgq5s9jIJH+G0CR0ZoHMB25Ln4X/bsCQbLOu21+IGYKSDVM5TIMLtkKUkERQMVWvnpOp1LZKir4dC0m7SW74wpA8+2b1IsURIr9ARYGJpCEv1Q1Wz/X3yTf6Mfey7992MjUc9HcgjgU01/+kYomoXHprzolk+22Gjfgo3a4dRIoTY82GO8kkUKiaWHvDkkVURCY5dpteLA05sk3Z9aRMYsNXPLeOOPfzTlDA0="
}
def test_index(self):
self._login('test@example.com')
with self.client as client:
response = client.get(url_for("console.index"))
self.assert_200(response)
def test_create_loads(self):
self._login('test@example.com')
with self.client as client:
response = client.get(url_for("console.create"))
self.assert_200(response)
def test_create_fails_capacity(self):
with self.client as client:
client.get(url_for("console.create"))
csrf_token = self.get_context_variable('csrf_token')
data = self.capsul_data
data['csrf-token'] = csrf_token
# Override MockHub.capacity_avaliable to always return False
with patch.object(MockHub, 'capacity_avaliable', return_value=False) as mock_method:
self.app.config["HUB_MODEL"] = MockHub()
client.post(url_for("console.create"), data=data)
capacity_message = \
'\n host(s) at capacity. no capsuls can be created at this time. sorry. \n '
self.assert_message_flashed(capacity_message, category='message')
self.assertEqual(
len(get_model().list_vms_for_account('test@example.com')),
0
)
mock_method.assert_called_with(512 * 1024 * 1024)
def test_create_fails_invalid(self):
with self.client as client:
client.get(url_for("console.create"))
csrf_token = self.get_context_variable('csrf_token')
data = self.capsul_data
data['csrf-token'] = csrf_token
data['os'] = ''
client.post(url_for("console.create"), data=data)
self.assert_message_flashed(
'OS is required',
category='message'
)
self.assertEqual(
len(get_model().list_vms_for_account('test@example.com')),
0
)
def test_create_succeeds(self):
with self.client as client:
client.get(url_for("console.create"))
csrf_token = self.get_context_variable('csrf_token')
data = self.capsul_data
data['csrf-token'] = csrf_token
response = client.post(url_for("console.create"), data=data)
# FIXME: mock create doesn't create, see #83
# vms = get_model().list_vms_for_account('test@example.com')
# self.assertEqual(
# len(vms),
# 1
# )
#
# vm_id = vms[0].id
#
# self.assertRedirects(
# response,
# url_for("console.index") + f'?{vm_id}'
# )
def test_keys_loads(self):
self._login('test@example.com')
with self.client as client:
response = client.get(url_for("console.ssh_api_keys"))
self.assert_200(response)
keys = self.get_context_variable('ssh_public_keys')
self.assertEqual(keys[0]['name'], 'key')
def test_keys_add_ssh_fails_invalid(self):
self._login('test@example.com')
with self.client as client:
client.get(url_for("console.ssh_api_keys"))
csrf_token = self.get_context_variable('csrf_token')
data = self.ssh_key_data
data['csrf-token'] = csrf_token
data_invalid_content = data
data_invalid_content['content'] = 'foo'
client.post(
url_for("console.ssh_api_keys"),
data=data_invalid_content
)
self.assert_message_flashed(
'Content must match "^(ssh|ecdsa)-[0-9A-Za-z+/_=@:. -]+$"',
category='message'
)
data_missing_content = data
data_missing_content['content'] = ''
client.post(url_for("console.ssh_api_keys"), data=data_missing_content)
self.assert_message_flashed(
'Content is required', category='message'
)
def test_keys_add_ssh_fails_duplicate(self):
self._login('test@example.com')
with self.client as client:
client.get(url_for("console.ssh_api_keys"))
csrf_token = self.get_context_variable('csrf_token')
data = self.ssh_key_data
data['csrf-token'] = csrf_token
data['name'] = 'key'
client.post(url_for("console.ssh_api_keys"), data=data)
self.assert_message_flashed(
'A key with that name already exists',
category='message'
)
data = self.ssh_key_data
data['csrf-token'] = csrf_token
data['name'] = 'key'
client.post(url_for("console.ssh_api_keys"), data=data)
self.assert_message_flashed(
'A key with that name already exists',
category='message'
)
def setUp(self):
self._login('test@example.com')
get_model().create_ssh_public_key('test@example.com', 'key', 'foo')
def tearDown(self):
get_model().delete_ssh_public_key('test@example.com', 'key')

View File

@ -0,0 +1,14 @@
from capsulflask.tests_base import BaseTestCase
class LandingTests(BaseTestCase):
#: Do not render templates, we're only testing logic here.
render_templates = False
def test_landing(self):
pages = ['/', 'pricing', 'faq', 'about-ssh', 'changelog', 'support']
with self.client as client:
for page in pages:
response = client.get(page)
self.assert_200(response)

View File

@ -0,0 +1,45 @@
from base64 import b64encode
import requests
from flask import url_for
from capsulflask.db import get_model
from capsulflask.tests_base import BaseLiveServerTestCase
class PublicAPITests(BaseLiveServerTestCase):
def test_server_is_up_and_running(self):
response = requests.get(self.get_server_url())
self.assertEqual(response.status_code, 200)
def test_capsul_create_succeeds(self):
response = requests.post(
self.get_server_url() +
url_for('publicapi.capsul_create'),
headers={'Authorization': self.token},
json={
'size': 'f1-xs',
'os': 'openbsd68',
'ssh_key_0': 'key'
}
)
self.assertEqual(response.status_code, 200)
# FIXME: mock create doesn't create, see #83
# vms = get_model().list_vms_for_account('test@example.com')
#
# self.assertEqual(
# len(vms),
# 1
# )
def setUp(self):
get_model().create_ssh_public_key('test@example.com', 'key', 'foo')
self.token = b64encode(
get_model().generate_api_token('test@example.com', 'apikey').encode('utf-8')
).decode('utf-8')
def tearDown(self):
get_model().delete_ssh_public_key('test@example.com', 'key')
get_model().delete_api_token('test@example.com', 1)

35
capsulflask/tests_base.py Normal file
View File

@ -0,0 +1,35 @@
import os
from nanoid import generate
from flask_testing import TestCase, LiveServerTestCase
from capsulflask import create_app
from capsulflask.db import get_model
class BaseSharedTestCase(object):
def create_app(self):
# Use default connection paramaters
os.environ['POSTGRES_CONNECTION_PARAMETERS'] = "host=localhost port=5432 user=postgres password=dev dbname=capsulflask_test"
os.environ['TESTING'] = '1'
os.environ['SPOKE_MODEL'] = 'mock'
os.environ['HUB_MODEL'] = 'mock'
return create_app()
def setUp(self):
pass
def tearDown(self):
pass
class BaseTestCase(BaseSharedTestCase, TestCase):
def _login(self, user_email):
get_model().login(user_email)
with self.client.session_transaction() as session:
session['account'] = user_email
session['csrf-token'] = generate()
class BaseLiveServerTestCase(BaseSharedTestCase, LiveServerTestCase):
pass